diff --git a/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java b/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java index 17fbb40e1b..a2895b4d55 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java @@ -64,14 +64,14 @@ public final class ReplaceBlackboardArtifactTagAction extends ReplaceTagAction() { @Override @@ -91,7 +91,7 @@ public final class ReplaceBlackboardArtifactTagAction extends ReplaceTagAction "# {1} - content obj id", "ReplaceContentTagAction.replaceTag.alert=Unable to replace tag {0} for {1}."}) @Override - protected void replaceTag(ContentTag oldTag, TagName newTagName, String comment) { + protected void replaceTag(ContentTag oldTag, TagName newTagName, String newComment) { new SwingWorker() { @Override @@ -84,7 +84,7 @@ public final class ReplaceContentTagAction extends ReplaceTagAction logger.log(Level.INFO, "Replacing tag {0} with tag {1} for artifact {2}", new Object[]{oldTag.getName().getDisplayName(), newTagName.getDisplayName(), oldTag.getContent().getName()}); //NON-NLS tagsManager.deleteContentTag(oldTag); - tagsManager.addContentTag(oldTag.getContent(), newTagName, comment); + tagsManager.addContentTag(oldTag.getContent(), newTagName, newComment); } catch (TskCoreException tskCoreException) { logger.log(Level.SEVERE, "Error replacing artifact tag", tskCoreException); //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java b/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java index 03d48f4012..79ca4f4739 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java @@ -141,7 +141,7 @@ abstract class ReplaceTagAction extends AbstractAction implements // Add action to replace the tag tagNameItem.addActionListener((ActionEvent event) -> { selectedTags.forEach((oldtag) -> { - replaceTag(oldtag, entry.getValue(), ""); + replaceTag(oldtag, entry.getValue(), oldtag.getComment()); }); }); @@ -178,7 +178,7 @@ abstract class ReplaceTagAction extends AbstractAction implements TagName newTagName = GetTagNameDialog.doDialog(); if (null != newTagName) { selectedTags.forEach((oldtag) -> { - replaceTag(oldtag, newTagName, ""); + replaceTag(oldtag, newTagName, oldtag.getComment()); }); } }); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 44bf52503f..9019ae162d 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -129,6 +129,7 @@ public class Case { private static final String EXPORT_FOLDER = "Export"; //NON-NLS private static final String LOG_FOLDER = "Log"; //NON-NLS private static final String REPORTS_FOLDER = "Reports"; //NON-NLS + private static final String CONFIG_FOLDER = "Config"; // NON-NLS private static final String TEMP_FOLDER = "Temp"; //NON-NLS private static final String MODULE_FOLDER = "ModuleOutput"; //NON-NLS private static final String CASE_ACTION_THREAD_NAME = "%s-case-action"; @@ -291,9 +292,10 @@ public class Case { */ ADDING_DATA_SOURCE_FAILED, /** - * A new data source has been added to the current case. The old value - * of the PropertyChangeEvent is null, the new value is the newly-added - * data source (type: Content). Cast the PropertyChangeEvent to + * A new data source or series of data sources have been added to the + * current case. The old value of the PropertyChangeEvent is null, the + * new value is the newly-added data source (type: Content). Cast the + * PropertyChangeEvent to * org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent to * access additional event data. */ @@ -1098,6 +1100,10 @@ public class Case { /* * Open the top components (windows within the main application * window). + * + * Note: If the core windows are not opened here, they will be + * opened via the DirectoryTreeTopComponent 'propertyChange()' + * method on a DATA_SOURCE_ADDED event. */ if (newCurrentCase.hasData()) { CoreComponentControl.openCoreWindows(); @@ -1351,6 +1357,16 @@ public class Case { return getOrCreateSubdirectory(REPORTS_FOLDER); } + /** + * Gets the full path to the config directory for this case, creating it if + * it does not exist. + * + * @return The config directory path. + */ + public String getConfigDirectory() { + return getOrCreateSubdirectory(CONFIG_FOLDER); + } + /** * Gets the full path to the module output directory for this case, creating * it if it does not exist. diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties b/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties index e9c1b5307c..266db3a64c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties @@ -1,6 +1,5 @@ OptionsCategory_Name_TagNamesOptions=Tags OptionsCategory_TagNames=TagNames -Blackboard.unableToIndexArtifact.error.msg=Unable to index blackboard artifact {0} TagNameDialog.title.text=New Tag TagNameDialog.JOptionPane.tagNameIllegalCharacters.message=Tag name may not contain any of the following symbols\: \\ \: * ? " < > | , ; TagNameDialog.JOptionPane.tagNameIllegalCharacters.title=Invalid character in tag name diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index dfeb4fbdcd..654ee7161e 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -105,7 +105,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { dbManager.updateAttributeInstanceComment(correlationAttribute); } } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); + logger.log(Level.SEVERE, "Error adding comment", ex); } } return centralRepoCommentDialog.getComment(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index ac286aa569..a81e336fac 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -58,6 +58,9 @@ abstract class AbstractSqlEamDb implements EamDb { protected int bulkArtifactsThreshold; private final Map> bulkArtifacts; + // Maximum length for the value column in the instance tables + static final int MAX_VALUE_LENGTH = 256; + // number of instances to keep in bulk queue before doing an insert. // Update Test code if this changes. It's hard coded there. static final int DEFAULT_BULK_THRESHHOLD = 1000; @@ -472,7 +475,8 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamDataSource.getCaseID() == -1) { throw new EamDbException("Case ID is -1"); } else if (eamDataSource.getID() != -1) { - throw new EamDbException("Database ID is already set in object"); + // This data source is already in the central repo + return; } Connection conn = connect(); @@ -632,6 +636,13 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamArtifact.getCorrelationValue() == null) { throw new EamDbException("Correlation value is null"); } + if (eamArtifact.getCorrelationValue().length() >= MAX_VALUE_LENGTH) { + throw new EamDbException("Artifact value too long for central repository." + + "\nCorrelationArtifact ID: " + eamArtifact.getID() + + "\nCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\nCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); + + } Connection conn = connect(); @@ -1056,27 +1067,50 @@ abstract class AbstractSqlEamDb implements EamDb { if (!eamArtifact.getCorrelationValue().isEmpty()) { if (eamInstance.getCorrelationCase() == null) { - throw new EamDbException("CorrelationAttributeInstance case is null"); + throw new EamDbException("CorrelationAttributeInstance case is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); } if (eamInstance.getCorrelationDataSource() == null) { - throw new EamDbException("CorrelationAttributeInstance data source is null"); + throw new EamDbException("CorrelationAttributeInstance data source is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); } if (eamInstance.getKnownStatus() == null) { - throw new EamDbException("CorrelationAttributeInstance known status is null"); + throw new EamDbException("CorrelationAttributeInstance known status is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue() + + "\n\tEam Instance: " + + "\n\t\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\t\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID()); } - bulkPs.setString(1, eamInstance.getCorrelationCase().getCaseUUID()); - bulkPs.setString(2, eamInstance.getCorrelationDataSource().getDeviceID()); - bulkPs.setInt(3, eamInstance.getCorrelationDataSource().getCaseID()); - bulkPs.setString(4, eamArtifact.getCorrelationValue()); - bulkPs.setString(5, eamInstance.getFilePath()); - bulkPs.setByte(6, eamInstance.getKnownStatus().getFileKnownValue()); - if ("".equals(eamInstance.getComment())) { - bulkPs.setNull(7, Types.INTEGER); + if (eamArtifact.getCorrelationValue().length() < MAX_VALUE_LENGTH) { + bulkPs.setString(1, eamInstance.getCorrelationCase().getCaseUUID()); + bulkPs.setString(2, eamInstance.getCorrelationDataSource().getDeviceID()); + bulkPs.setInt(3, eamInstance.getCorrelationDataSource().getCaseID()); + bulkPs.setString(4, eamArtifact.getCorrelationValue()); + bulkPs.setString(5, eamInstance.getFilePath()); + bulkPs.setByte(6, eamInstance.getKnownStatus().getFileKnownValue()); + if ("".equals(eamInstance.getComment())) { + bulkPs.setNull(7, Types.INTEGER); + } else { + bulkPs.setString(7, eamInstance.getComment()); + } + bulkPs.addBatch(); } else { - bulkPs.setString(7, eamInstance.getComment()); + logger.log(Level.WARNING, ("Artifact value too long for central repository." + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()) + + "\n\tEam Instance: " + + "\n\t\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\t\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID() + + "\n\t\tFilePath: " + eamInstance.getFilePath()); } - bulkPs.addBatch(); } } } @@ -1821,11 +1855,13 @@ abstract class AbstractSqlEamDb implements EamDb { return 0 < badInstances; } + /** * Process the Artifact instance in the EamDb * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance + * * @throws EamDbException */ @Override diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java index 8cf07a42e9..84dce834a1 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java @@ -67,6 +67,7 @@ public class CorrelationDataSource implements Serializable { /** * Create a CorrelationDataSource object from a TSK Content object. + * This will add it to the central repository. * * @param correlationCase the current CorrelationCase used for ensuring * uniqueness of DataSource diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 55958287d5..a3d49d6ba3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -450,10 +450,10 @@ final class CaseEventListener implements PropertyChangeListener { correlationCase = dbManager.newCase(openCase); } if (null == dbManager.getDataSource(correlationCase, deviceId)) { - dbManager.newDataSource(CorrelationDataSource.fromTSKDataSource(correlationCase, newDataSource)); + CorrelationDataSource.fromTSKDataSource(correlationCase, newDataSource); } } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS + LOGGER.log(Level.SEVERE, "Error adding new data source to the central repository", ex); //NON-NLS } catch (TskCoreException | TskDataException ex) { LOGGER.log(Level.SEVERE, "Error getting data source from DATA_SOURCE_ADDED event content.", ex); //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 46ebe75541..e3cfad9fe8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -276,12 +276,12 @@ public class IngestEventsListener { } } if (FALSE == eamArtifacts.isEmpty()) { - try { - for (CorrelationAttribute eamArtifact : eamArtifacts) { + for (CorrelationAttribute eamArtifact : eamArtifacts) { + try { dbManager.addArtifact(eamArtifact); + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error adding artifact to database.", ex); //NON-NLS } - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS } } // DATA_ADDED } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributesSearchResultsViewerTable.java index 8aab16c667..1e54ff80fd 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributesSearchResultsViewerTable.java @@ -23,6 +23,8 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.table.TableColumn; import javax.swing.table.TableColumnModel; import org.openide.util.NbBundle; @@ -36,10 +38,14 @@ import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; * rows and looking for wide cells. Rather, we just pick some reasonable values. */ public class CommonAttributesSearchResultsViewerTable extends DataResultViewerTable { - + private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; + + private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); + private static final int DEFAULT_WIDTH = 100; + static { Map map = new HashMap<>(); map.put(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), 260); @@ -50,10 +56,10 @@ public class CommonAttributesSearchResultsViewerTable extends DataResultViewerTa map.put(Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), 100); map.put(Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), 130); map.put(Bundle.CommonFilesSearchResultsViewerTable_tagsColLbl1(), 300);; - + COLUMN_WIDTHS = Collections.unmodifiableMap(map); } - + @NbBundle.Messages({ "CommonFilesSearchResultsViewerTable.noDescText= ", "CommonFilesSearchResultsViewerTable.filesColLbl=Files", @@ -66,18 +72,24 @@ public class CommonAttributesSearchResultsViewerTable extends DataResultViewerTa "CommonFilesSearchResultsViewerTable.tagsColLbl1=Tags" }) @Override - protected void setColumnWidths(){ + protected void setColumnWidths() { TableColumnModel model = this.getColumnModel(); - + Enumeration columnsEnumerator = model.getColumns(); - while(columnsEnumerator.hasMoreElements()){ - + while (columnsEnumerator.hasMoreElements()) { + TableColumn column = columnsEnumerator.nextElement(); - + final String headerValue = column.getHeaderValue().toString(); - final Integer get = COLUMN_WIDTHS.get(headerValue); - - column.setPreferredWidth(get); + + final Integer defaultWidth = COLUMN_WIDTHS.get(headerValue); + + if(defaultWidth == null){ + column.setPreferredWidth(DEFAULT_WIDTH); + LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue)); + } else { + column.setPreferredWidth(defaultWidth); + } } } } diff --git a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java index bf26dc8295..06ad27962a 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java @@ -88,7 +88,21 @@ final public class CoreComponentControl { TopComponent directoryTree = null; TopComponent favorites = null; final WindowManager windowManager = WindowManager.getDefault(); + + // Set the UI selections to null before closing the top components. + // Otherwise it may experience errors trying to load data for the closed case. for (Mode mode : windowManager.getModes()) { + for (TopComponent tc : windowManager.getOpenedTopComponents(mode)) { + if(tc instanceof DataContent) { + ((DataContent) tc).setNode(null); + } else if(tc instanceof DataResult) { + ((DataResult) tc).setNode(null); + } + } + } + + for (Mode mode : windowManager.getModes()) { + for (TopComponent tc : windowManager.getOpenedTopComponents(mode)) { String tcName = tc.getName(); @@ -105,7 +119,7 @@ final public class CoreComponentControl { } } } - + if (directoryTree != null) { directoryTree.close(); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java index 6a304d7b86..aa1d0a04ec 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java @@ -249,7 +249,7 @@ public final class FileTypes implements AutopsyVisitableItem { if (typesRoot.showCounts) { //only show "(counting...)" the first time, otherwise it is distracting. setDisplayName(getDisplayNameBase() + ((childCount < 0) ? Bundle.FileTypes_bgCounting_placeholder() - : ("(" + childCount + ")"))); //NON-NLS + : (" (" + childCount + ")"))); //NON-NLS new SwingWorker() { @Override protected Long doInBackground() throws Exception { diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties b/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties index 0e7f3cf315..926c4b4626 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties @@ -120,3 +120,8 @@ AddExternalViewerRulePanel.exePathTextField.text= AddExternalViewerRulePanel.exePathLabel.text=Path of the program to use for files with this type or extension AddExternalViewerRulePanel.extRadioButton.text=Extension DirectoryTreeTopComponent.groupByDatasourceCheckBox.text=Group by Data Source +GroupDataSourcesDialog.dataSourceCountLabel.text=jLabel1 +GroupDataSourcesDialog.queryLabel.text=Would you like to group by data source for faster loading? +GroupDataSourcesDialog.yesButton.text=Yes +GroupDataSourcesDialog.noButton.text=No +GroupDataSourcesDialog.title=Group by Data Source? \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 7db201f00b..95ba2f110c 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -24,6 +24,11 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyVetoException; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -35,6 +40,7 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.prefs.PreferenceChangeEvent; import java.util.prefs.PreferenceChangeListener; +import java.util.Properties; import javax.swing.Action; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; @@ -62,6 +68,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataExplorer; import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent; import org.sleuthkit.autopsy.corecomponents.TableFilterNode; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.ModuleSettings; import org.sleuthkit.autopsy.datamodel.ArtifactNodeSelectionInfo; import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode; import org.sleuthkit.autopsy.datamodel.CreditCards; @@ -103,6 +110,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private static final Logger LOGGER = Logger.getLogger(DirectoryTreeTopComponent.class.getName()); private AutopsyTreeChildrenFactory autopsyTreeChildrenFactory; private Children autopsyTreeChildren; + private static final long DEFAULT_DATASOURCE_GROUPING_THRESHOLD = 5; // Threshold for prompting the user about grouping by data source + private static final String GROUPING_THRESHOLD_NAME = "GroupDataSourceThreshold"; + private static final String SETTINGS_FILE = "CasePreferences.properties"; //NON-NLS /** * the constructor @@ -125,7 +135,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat this.forwardList = new LinkedList<>(); backButton.setEnabled(false); forwardButton.setEnabled(false); - + groupByDatasourceCheckBox.setSelected(UserPreferences.groupItemsInTreeByDatasource()); } @@ -149,7 +159,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } }); - + Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.DATA_SOURCE_ADDED), this); this.em.addPropertyChangeListener(this); IngestManager.getInstance().addIngestJobEventListener(this); @@ -370,6 +380,51 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat return TopComponent.PERSISTENCE_NEVER; } + /** + * Ask the user if they want to group by data source when opening a large + * case. + * + * @param currentCase + * @param dataSourceCount + */ + private void promptForDataSourceGrouping(Case currentCase, int dataSourceCount) { + Path settingsFile = Paths.get(currentCase.getConfigDirectory(), SETTINGS_FILE); //NON-NLS + if (settingsFile.toFile().exists()) { + // Read the setting + try (InputStream inputStream = Files.newInputStream(settingsFile)) { + Properties props = new Properties(); + props.load(inputStream); + if (props.getProperty("groupByDataSource", "false").equals("true")) { + UserPreferences.setGroupItemsInTreeByDatasource(true); + groupByDatasourceCheckBox.setSelected(true); + } + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Error reading settings file", ex); + } + } else { + GroupDataSourcesDialog dialog = new GroupDataSourcesDialog(dataSourceCount); + dialog.display(); + if (dialog.groupByDataSourceSelected()) { + UserPreferences.setGroupItemsInTreeByDatasource(true); + groupByDatasourceCheckBox.setSelected(true); + } + + // Save the response + Properties props = new Properties(); + if (dialog.groupByDataSourceSelected()) { + props.setProperty("groupByDataSource", "true"); + } else { + props.setProperty("groupByDataSource", "false"); + } + + try (OutputStream fos = Files.newOutputStream(settingsFile)) { + props.store(fos, ""); //NON-NLS + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Error writing settings file", ex); + } + } + } + /** * Called only when top component was closed on all workspaces before and * now is opened for the first time on some workspace. The intent is to @@ -377,6 +432,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * existing workspaces. Subclasses will usually perform initializing tasks * here. */ + @NbBundle.Messages({"# {0} - dataSourceCount", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.text=This case contains {0} data sources. Would you like to group by data source for faster loading?", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.title=Group by data source?"}) @Override public void componentOpened() { // change the cursor to "waiting cursor" for this operation @@ -392,9 +450,34 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat if (null == currentCase || currentCase.hasData() == false) { getTree().setRootVisible(false); // hide the root } else { + // If the case contains a lot of data sources, and they aren't already grouping + // by data source, give the user the option to do so before loading the tree. + if (RuntimeProperties.runningWithGUI()) { + long threshold = DEFAULT_DATASOURCE_GROUPING_THRESHOLD; + if (ModuleSettings.settingExists(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME)) { + try { + threshold = Long.parseLong(ModuleSettings.getConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME)); + } catch (NumberFormatException ex) { + LOGGER.log(Level.SEVERE, "Group data sources threshold is not a number", ex); + } + } else { + ModuleSettings.setConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME, String.valueOf(threshold)); + } + + try { + int dataSourceCount = currentCase.getDataSources().size(); + if (!UserPreferences.groupItemsInTreeByDatasource() + && dataSourceCount > threshold) { + promptForDataSourceGrouping(currentCase, dataSourceCount); + } + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error loading data sources", ex); + } + } + // if there's at least one image, load the image and open the top componen autopsyTreeChildrenFactory = new AutopsyTreeChildrenFactory(); - autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); + autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); Node root = new AbstractNode(autopsyTreeChildren) { //JIRA-2807: What is the point of these overrides? /** @@ -453,9 +536,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat tree.collapseNode(views); } /* - * JIRA-2806: What is this supposed to do? Right now it selects - * the data sources node, but the comment seems to indicate - * it is supposed to select the first datasource. + * JIRA-2806: What is this supposed to do? Right now it + * selects the data sources node, but the comment seems to + * indicate it is supposed to select the first datasource. */ // select the first image node, if there is one // (this has to happen after dataResult is opened, because the event @@ -484,7 +567,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // dataResult active) try { Node[] selections = get(); - if (selections != null && selections.length > 0){ + if (selections != null && selections.length > 0) { em.setSelectedNodes(selections); } } catch (PropertyVetoException ex) { @@ -592,15 +675,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } /** - * The "listener" that listens to any changes made in the Case.java class. - * It will do something based on the changes in the Case.java class. + * The "listener" that monitors changes made in the Case class. This serves + * the purpose of keeping the UI in sync with the data as it changes. * - * @param evt the property change event + * @param event The property change event. */ @Override - public void propertyChange(PropertyChangeEvent evt) { + public void propertyChange(PropertyChangeEvent event) { if (RuntimeProperties.runningWithGUI()) { - String changed = evt.getPropertyName(); + String changed = event.getPropertyName(); if (changed.equals(Case.Events.CURRENT_CASE.toString())) { // changed current case // When a case is closed, the old value of this property is the // closed Case object and the new value is null. When a case is @@ -610,15 +693,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // opened events instead of property change events would be a better // solution. Either way, more probably needs to be done to clean up // data model objects when a case is closed. - if (evt.getOldValue() != null && evt.getNewValue() == null) { + if (event.getOldValue() != null && event.getNewValue() == null) { // The current case has been closed. Reset the ExplorerManager. SwingUtilities.invokeLater(() -> { Node emptyNode = new AbstractNode(Children.LEAF); em.setRootContext(emptyNode); }); - } else if (evt.getNewValue() != null) { + } else if (event.getNewValue() != null) { // A new case has been opened. Reset the ExplorerManager. - Case newCase = (Case) evt.getNewValue(); + Case newCase = (Case) event.getNewValue(); final String newCaseName = newCase.getName(); SwingUtilities.invokeLater(() -> { em.getRootContext().setName(newCaseName); @@ -642,20 +725,27 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * already closed. */ try { - Case currentCase = Case.getCurrentCaseThrows(); - // We only need to trigger openCoreWindows() when the - // first data source is added. - if (currentCase.getDataSources().size() == 1) { + Case.getCurrentCaseThrows(); + /* + * In case the Case 'updateGUIForCaseOpened()' method hasn't + * already done so, open the tree and all other core + * windows. + * + * TODO: (JIRA-4053) DirectoryTreeTopComponent should not be + * responsible for opening core windows. Consider moving + * this elsewhere. + */ + if (!this.isOpened()) { SwingUtilities.invokeLater(CoreComponentControl::openCoreWindows); } - } catch (NoCurrentCaseException | TskCoreException notUsed) { + } catch (NoCurrentCaseException notUsed) { /** * Case is closed, do nothing. */ } } // change in node selection else if (changed.equals(ExplorerManager.PROP_SELECTED_NODES)) { - respondSelection((Node[]) evt.getOldValue(), (Node[]) evt.getNewValue()); + respondSelection((Node[]) event.getOldValue(), (Node[]) event.getNewValue()); } else if (changed.equals(IngestManager.IngestModuleEvent.DATA_ADDED.toString())) { // nothing to do here. // all nodes should be listening for these events and update accordingly. @@ -800,7 +890,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat /** * Rebuilds the autopsy tree. - * + * * Does nothing if there is no open case. */ private void rebuildTree() { @@ -813,9 +903,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat return; } if (null == currentCase || currentCase.hasData() == false) { - return; + return; } - + // refresh all children of the root. autopsyTreeChildrenFactory.refreshChildren(); @@ -842,14 +932,14 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } }.execute(); } - + /** * Selects the first node in the tree. - * + * */ - private void selectFirstChildNode () { + private void selectFirstChildNode() { Children rootChildren = em.getRootContext().getChildren(); - + if (rootChildren.getNodesCount() > 0) { Node firstNode = rootChildren.getNodeAt(0); if (firstNode != null) { @@ -858,6 +948,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } } + /** * Set the selected node using a path to a previously selected node. * diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java index 96d7973d8a..1ed54df58d 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.filesearch; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.io.File; import java.util.ArrayList; import java.util.Collections; @@ -29,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; -import javax.swing.JList; import javax.swing.event.ListSelectionEvent; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -47,38 +44,28 @@ public class DataSourcePanel extends javax.swing.JPanel { private static final Logger logger = Logger.getLogger(DataSourcePanel.class.getName()); private static final long serialVersionUID = 1L; private final Map dataSourceMap = new HashMap<>(); - private final List toolTipList = new ArrayList<>(); /** * Creates new form DataSourcePanel */ public DataSourcePanel() { initComponents(); - this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { - firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); - }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - if (evt.getSource() instanceof JList) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(toolTipList.get(index)); - } - } - } - }); + if (this.dataSourceList.getModel().getSize() > 1) { + this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { + firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); + }); + } else { + /* + * Disable data source filtering since there aren't multiple data + * sources to choose from. + */ + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceList.setEnabled(false); + } } /** - * Get dataSourceMap with object id and data source display name. Add the data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. * * @return The list of data source name */ @@ -93,8 +80,7 @@ public class DataSourcePanel extends javax.swing.JPanel { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); - dataSourceMap.put(ds.getId(), displayName); - toolTipList.add(dsName); + dataSourceMap.put(ds.getId(), displayName); dsList.add(displayName); } } catch (NoCurrentCaseException ex) { @@ -107,7 +93,8 @@ public class DataSourcePanel extends javax.swing.JPanel { /** * Get a set of data source object ids that are selected. - * @return A set of selected object ids. + * + * @return A set of selected object ids. */ Set getDataSourcesSelected() { Set dataSourceObjIdSet = new HashSet<>(); @@ -124,6 +111,7 @@ public class DataSourcePanel extends javax.swing.JPanel { /** * Is dataSourceCheckBox selected + * * @return true if the dataSoureCheckBox is selected */ boolean isSelected() { @@ -131,7 +119,8 @@ public class DataSourcePanel extends javax.swing.JPanel { } /** - * Enable the dsList and dataSourceNoteLable if the dataSourceCheckBox is checked. + * Enable the dsList and dataSourceNoteLable if the dataSourceCheckBox is + * checked. */ final void setComponentsEnabled() { boolean enabled = this.isSelected(); diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index b69d84b629..50dfe318c3 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -556,17 +556,29 @@ class ReportHTML implements TableReportModule { } /** - * Add a row to the current table. + * Add a row to the current table, escaping the text to be contained in the + * row. * * @param row values for each cell in the row */ @Override public void addRow(List row) { + addRow(row, true); + } + + /** + * Add a row to the current table. + * + * @param row values for each cell in the row + * @param escapeText whether or not the text of the row should be escaped, + * true for escaped, false for not escaped + */ + private void addRow(List row, boolean escapeText) { StringBuilder builder = new StringBuilder(); builder.append("\t\n"); //NON-NLS for (String cell : row) { - String escapeHTMLCell = EscapeUtil.escapeHtml(cell); - builder.append("\t\t").append(escapeHTMLCell).append("\n"); //NON-NLS + String cellText = escapeText ? EscapeUtil.escapeHtml(cell) : cell; + builder.append("\t\t").append(cellText).append("\n"); //NON-NLS } builder.append("\t\n"); //NON-NLS rowCount++; @@ -593,7 +605,7 @@ class ReportHTML implements TableReportModule { public void addRowWithTaggedContentHyperlink(List row, ContentTag contentTag) { Content content = contentTag.getContent(); if (content instanceof AbstractFile == false) { - addRow(row); + addRow(row, true); return; } AbstractFile file = (AbstractFile) content; @@ -647,7 +659,7 @@ class ReportHTML implements TableReportModule { int pages = 1; for (Content content : images) { if (currentRow.size() == THUMBNAIL_COLUMNS) { - addRow(currentRow); + addRow(currentRow, false); currentRow.clear(); } @@ -727,7 +739,7 @@ class ReportHTML implements TableReportModule { // Finish out the row. currentRow.add(""); } - addRow(currentRow); + addRow(currentRow, false); } // manually set rowCount to be the total number of images. @@ -1094,8 +1106,8 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS summary.append(writeSummaryCaseDetails()); summary.append(writeSummaryImageInfo()); - summary.append(writeSummarySoftwareInfo(skCase,ingestJobs)); - summary.append(writeSummaryIngestHistoryInfo(skCase,ingestJobs)); + summary.append(writeSummarySoftwareInfo(skCase, ingestJobs)); + summary.append(writeSummaryIngestHistoryInfo(skCase, ingestJobs)); if (generatorLogoSet) { summary.append("
\n"); //NON-NLS summary.append("\n"); //NON-NLS @@ -1126,14 +1138,13 @@ class ReportHTML implements TableReportModule { } } } - + /** * Write the case details section of the summary for this report. - * + * * @return StringBuilder updated html report with case details */ - - private StringBuilder writeSummaryCaseDetails(){ + private StringBuilder writeSummaryCaseDetails() { StringBuilder summary = new StringBuilder(); String caseName = currentCase.getDisplayName(); String caseNumber = currentCase.getNumber(); @@ -1146,40 +1157,39 @@ class ReportHTML implements TableReportModule { imagecount = 0; } summary.append("
\n"); //NON-NLS - if (agencyLogoSet) { - summary.append("
\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - } - final String align = agencyLogoSet ? "right" : "left"; //NON-NLS NON-NLS - summary.append("
\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseName")) //NON-NLS - .append("").append(caseName).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseNum")) //NON-NLS - .append("").append(!caseNumber.isEmpty() ? caseNumber : NbBundle //NON-NLS - .getMessage(this.getClass(), "ReportHTML.writeSum.noCaseNum")).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.examiner")).append("") //NON-NLS - .append(!examiner.isEmpty() ? examiner : NbBundle - .getMessage(this.getClass(), "ReportHTML.writeSum.noExaminer")) - .append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.numImages")) //NON-NLS - .append("").append(imagecount).append("
\n"); //NON-NLS + if (agencyLogoSet) { + summary.append("
\n"); //NON-NLS + summary.append("\n"); //NON-NLS summary.append("
\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - return summary; + } + final String align = agencyLogoSet ? "right" : "left"; //NON-NLS NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseName")) //NON-NLS + .append("").append(caseName).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseNum")) //NON-NLS + .append("").append(!caseNumber.isEmpty() ? caseNumber : NbBundle //NON-NLS + .getMessage(this.getClass(), "ReportHTML.writeSum.noCaseNum")).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.examiner")).append("") //NON-NLS + .append(!examiner.isEmpty() ? examiner : NbBundle + .getMessage(this.getClass(), "ReportHTML.writeSum.noExaminer")) + .append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.numImages")) //NON-NLS + .append("").append(imagecount).append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + return summary; } - + /** * Write the Image Information section of the summary for this report. - * + * * @return StringBuilder updated html report with Image Information */ - private StringBuilder writeSummaryImageInfo() { StringBuilder summary = new StringBuilder(); summary.append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.imageInfoHeading")); @@ -1208,13 +1218,12 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS return summary; } - + /** * Write the software information section of the summary for this report. - * + * * @return StringBuilder updated html report with software information */ - private StringBuilder writeSummarySoftwareInfo(SleuthkitCase skCase, List ingestJobs) { StringBuilder summary = new StringBuilder(); summary.append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.softwareInfoHeading")); @@ -1244,13 +1253,12 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS return summary; } - + /** * Write the Ingest History section of the summary for this report. - * + * * @return StringBuilder updated html report with ingest history */ - private StringBuilder writeSummaryIngestHistoryInfo(SleuthkitCase skCase, List ingestJobs) { StringBuilder summary = new StringBuilder(); try { @@ -1304,4 +1312,4 @@ class ReportHTML implements TableReportModule { + thumbFile.getName(); } -} \ No newline at end of file +} diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 9d5d41c836..f87851fe87 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -44,6 +44,7 @@ import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import static org.sleuthkit.autopsy.casemodule.services.TagsManager.getNotableTagLabel; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -53,6 +54,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute.Type; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -538,6 +540,65 @@ class TableReportGenerator { logger.log(Level.SEVERE, "Exception while getting open case: ", ex); //NON-NLS return; } + + // Get a list of all selected tag IDs + String tagIDList = ""; + if( ! tagNamesFilter.isEmpty()) { + try { + Map tagNamesMap = Case.getCurrentCaseThrows().getServices().getTagsManager().getDisplayNamesToTagNamesMap(); + for(String tagDisplayName : tagNamesFilter) { + if(tagNamesMap.containsKey(tagDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(tagDisplayName).getId(); + } else { + // If the tag name ends with "(Notable)", try stripping that off + if(tagDisplayName.endsWith(getNotableTagLabel())) { + String editedDisplayName = tagDisplayName.substring(0, tagDisplayName.length() - getNotableTagLabel().length()); + if(tagNamesMap.containsKey(editedDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(editedDisplayName).getId(); + } + } + } + } + } catch (NoCurrentCaseException | TskCoreException ex) { + logger.log(Level.SEVERE, "Exception while getting tag info - proceeding without tag filter: ", ex); //NON-NLS + tagIDList = ""; + } + } + + // Check if there are any ad-hoc results + String adHocCountQuery = "SELECT COUNT(*) FROM " + //NON-NLS + "(SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 ";//NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + adHocCountQuery += "WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += " AND (art.artifact_id = tag.artifact_id) AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + adHocCountQuery += "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ")) "; //NON-NLS + + int adHocCount = 0; + try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(adHocCountQuery)) { + ResultSet adHocCountResultSet = dbQuery.getResultSet(); + if (adHocCountResultSet.next()) { + adHocCount = adHocCountResultSet.getInt(1); //NON-NLS + } else { + throw new TskCoreException("Error counting ad hoc keywords"); + } + } catch (TskCoreException | SQLException ex) { + errorList.add(NbBundle.getMessage(this.getClass(), "ReportGenerator.errList.failedQueryKWLists")); + logger.log(Level.SEVERE, "Failed to count ad hoc searches with query " + adHocCountQuery, ex); //NON-NLS + return; + } + + // Create the query to get the keyword list names if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { @@ -546,16 +607,25 @@ class TableReportGenerator { String keywordListQuery = "SELECT att.value_text AS list " + //NON-NLS - "FROM blackboard_attributes AS att, blackboard_artifacts AS art " - + //NON-NLS - "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + "FROM blackboard_attributes AS att, blackboard_artifacts AS art "; // NON-NLS + if(! tagIDList.isEmpty()) { + keywordListQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + keywordListQuery += "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + //NON-NLS "AND art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + " " + //NON-NLS - "AND att.artifact_id = art.artifact_id " - + //NON-NLS - "GROUP BY list " + orderByClause; //NON-NLS + "AND att.artifact_id = art.artifact_id "; + if (! tagIDList.isEmpty()) { + keywordListQuery += "AND (art.artifact_id = tag.artifact_id) " + //NON-NLS + "AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + if (adHocCount > 0) { + keywordListQuery += " UNION SELECT \"\" AS list "; + } + keywordListQuery += "GROUP BY list " + orderByClause; //NON-NLS + // Make the table of contents links for each list type try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { ResultSet listsRs = dbQuery.getResultSet(); List lists = new ArrayList<>(); @@ -579,6 +649,7 @@ class TableReportGenerator { return; } + // Query for keywords, grouped by list if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att3.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS + "convert_to(att1.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS @@ -588,9 +659,10 @@ class TableReportGenerator { } else { orderByClause = "ORDER BY list ASC, keyword ASC, parent_path ASC, name ASC, preview ASC"; //NON-NLS } - // Query for keywords, grouped by list - String keywordsQuery - = "SELECT art.artifact_id, art.obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + + // Query for keywords that are part of a list + String keywordListsQuery + = "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + //NON-NLS "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, blackboard_attributes AS att3, tsk_files AS f " + //NON-NLS @@ -608,9 +680,24 @@ class TableReportGenerator { + //NON-NLS "AND (att3.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ") " + //NON-NLS - "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " - + //NON-NLS - orderByClause; //NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; + + // Query for keywords that are not part of a list + String keywordAdHocQuery = + "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \"\" AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS + "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, tsk_files AS f " + // NON-NLS + "WHERE " + // NON-NLS + " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " + // NON-NLS + "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + "))) " + //NON-NLS + "AND (att1.artifact_id = art.artifact_id) " + //NON-NLS + "AND (att2.artifact_id = art.artifact_id) " + //NON-NLS + "AND (f.obj_id = art.obj_id) " + //NON-NLS + "AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID() + ") " + // NON-NLS + "AND (att2.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID() + ") " + // NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + + String keywordsQuery = keywordListsQuery + " UNION " + keywordAdHocQuery + orderByClause; try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordsQuery)) { ResultSet resultSet = dbQuery.getResultSet(); @@ -1623,14 +1710,15 @@ class TableReportGenerator { private HashSet getUniqueTagNames(long artifactId) throws TskCoreException { HashSet uniqueTagNames = new HashSet<>(); - String query = "SELECT display_name, artifact_id FROM tag_names AS tn, blackboard_artifact_tags AS bat " + String query = "SELECT display_name, artifact_id, knownStatus FROM tag_names AS tn, blackboard_artifact_tags AS bat " + //NON-NLS "WHERE tn.tag_name_id = bat.tag_name_id AND bat.artifact_id = " + artifactId; //NON-NLS try (SleuthkitCase.CaseDbQuery dbQuery = Case.getCurrentCaseThrows().getSleuthkitCase().executeQuery(query)) { ResultSet tagNameRows = dbQuery.getResultSet(); while (tagNameRows.next()) { - uniqueTagNames.add(tagNameRows.getString("display_name")); //NON-NLS + String notableString = tagNameRows.getInt("knownStatus") == TskData.FileKnown.BAD.ordinal() ? getNotableTagLabel() : ""; + uniqueTagNames.add(tagNameRows.getString("display_name") + notableString); //NON-NLS } } catch (TskCoreException | SQLException | NoCurrentCaseException ex) { throw new TskCoreException("Error getting tag names for artifact: ", ex); diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java index f557641df0..69abc2810f 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java @@ -104,16 +104,36 @@ class IntraCaseUtils { } void setUp() { - CaseUtils.createAsCurrentCase(this.caseName); + this.createAsCurrentCase(); final ImageDSProcessor imageDSProcessor = new ImageDSProcessor(); - IngestUtils.addDataSource(imageDSProcessor, imagePath1); - IngestUtils.addDataSource(imageDSProcessor, imagePath2); - IngestUtils.addDataSource(imageDSProcessor, imagePath3); + this.addImageOne(imageDSProcessor); + this.addImageTwo(imageDSProcessor); + this.addImageThree(imageDSProcessor); + this.addImageFour(imageDSProcessor); + } + + void addImageFour(final ImageDSProcessor imageDSProcessor) { IngestUtils.addDataSource(imageDSProcessor, imagePath4); } + void addImageThree(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath3); + } + + void addImageTwo(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath2); + } + + void addImageOne(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath1); + } + + void createAsCurrentCase() { + CaseUtils.createAsCurrentCase(this.caseName); + } + Map getDataSourceMap() throws NoCurrentCaseException, TskCoreException, SQLException { return this.dataSourceLoader.getDataSourceMap(); } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index 9830cecbe9..e62c58343c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -33,7 +33,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.LocalDiskDSProcessor; import org.sleuthkit.autopsy.casemodule.LocalFilesDSProcessor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import static org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java index 87c805fa12..1e1b82d8e7 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java @@ -338,7 +338,7 @@ final class AutoIngestAdminActions { dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); AutoIngestManager.CaseDeletionResult result = dashboard.getMonitor().deleteCase(job); - dashboard.getCompletedJobsPanel().refresh(new AutoIngestNodeRefreshEvents.RefreshChildrenEvent(dashboard.getMonitor().getJobsSnapshot())); + dashboard.getCompletedJobsPanel().refresh(new AutoIngestNodeRefreshEvents.RefreshChildrenEvent(dashboard.getMonitor())); dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); if (AutoIngestManager.CaseDeletionResult.FAILED == result) { JOptionPane.showMessageDialog(dashboard, diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java index 49e9cdbd17..031aa64daf 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java @@ -38,7 +38,6 @@ import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.core.ServicesMonitor; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeRefreshEvents.RefreshChildrenEvent; /** @@ -257,7 +256,7 @@ final class AutoIngestDashboard extends JPanel implements Observer { @Override public void update(Observable observable, Object arg) { - if (arg instanceof JobsSnapshot) { + if (arg == null ) { EventQueue.invokeLater(() -> { refreshTables(); }); @@ -271,9 +270,9 @@ final class AutoIngestDashboard extends JPanel implements Observer { * @param nodeStateSnapshot The jobs snapshot. */ void refreshTables() { - pendingJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); - runningJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); - completedJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); + pendingJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); + runningJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); + completedJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); } /** diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java index a7b2ad6116..552e44834c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java @@ -33,7 +33,6 @@ import org.openide.nodes.Node; import org.openide.nodes.Sheet; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.datamodel.NodeProperty; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; import org.sleuthkit.autopsy.guiutils.DurationCellRenderer; import org.sleuthkit.autopsy.guiutils.StatusIconCellRenderer; @@ -61,13 +60,13 @@ final class AutoIngestJobsNode extends AbstractNode { /** * Construct a new AutoIngestJobsNode. * - * @param snapshot the snapshot which contains the AutoIngestJobs + * @param monitor the monitor which gives access to the AutoIngestJobs * @param status the status of the jobs being displayed * @param eventBus the event bus which will be used to send and receive * refresh events */ - AutoIngestJobsNode(JobsSnapshot jobsSnapshot, AutoIngestJobStatus status, EventBus eventBus) { - super(Children.create(new AutoIngestNodeChildren(jobsSnapshot, status, eventBus), false)); + AutoIngestJobsNode(AutoIngestMonitor monitor, AutoIngestJobStatus status, EventBus eventBus) { + super(Children.create(new AutoIngestNodeChildren(monitor, status, eventBus), false)); refreshChildrenEventBus = eventBus; } @@ -84,7 +83,7 @@ final class AutoIngestJobsNode extends AbstractNode { static final class AutoIngestNodeChildren extends ChildFactory { private final AutoIngestJobStatus autoIngestJobStatus; - private JobsSnapshot jobsSnapshot; + private AutoIngestMonitor monitor; private final RefreshChildrenSubscriber refreshChildrenSubscriber = new RefreshChildrenSubscriber(); private final EventBus refreshEventBus; @@ -92,13 +91,13 @@ final class AutoIngestJobsNode extends AbstractNode { * Create children nodes for the AutoIngestJobsNode which will each * represent a single AutoIngestJob * - * @param snapshot the snapshot which contains the AutoIngestJobs + * @param monitor the monitor which gives access to the AutoIngestJobs * @param status the status of the jobs being displayed * @param eventBus the event bus which the class registers to for * refresh events */ - AutoIngestNodeChildren(JobsSnapshot snapshot, AutoIngestJobStatus status, EventBus eventBus) { - jobsSnapshot = snapshot; + AutoIngestNodeChildren(AutoIngestMonitor monitor, AutoIngestJobStatus status, EventBus eventBus) { + this.monitor = monitor; autoIngestJobStatus = status; refreshEventBus = eventBus; refreshChildrenSubscriber.register(refreshEventBus); @@ -109,14 +108,14 @@ final class AutoIngestJobsNode extends AbstractNode { List jobs; switch (autoIngestJobStatus) { case PENDING_JOB: - jobs = jobsSnapshot.getPendingJobs(); + jobs = monitor.getPendingJobs(); Collections.sort(jobs); break; case RUNNING_JOB: - jobs = jobsSnapshot.getRunningJobs(); + jobs = monitor.getRunningJobs(); break; case COMPLETED_JOB: - jobs = jobsSnapshot.getCompletedJobs(); + jobs = monitor.getCompletedJobs(); break; default: jobs = new ArrayList<>(); @@ -167,7 +166,7 @@ final class AutoIngestJobsNode extends AbstractNode { //Ignore netbeans suggesting this isn't being used, it is used behind the scenes by the EventBus //RefreshChildrenEvents can change which children are present however //RefreshJobEvents and RefreshCaseEvents can still change the order we want to display them in - jobsSnapshot = refreshEvent.getJobsSnapshot(); + monitor = refreshEvent.getMonitor(); refresh(true); } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java index a3cdc1c51d..fb6f79ab0c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java @@ -171,7 +171,7 @@ final class AutoIngestJobsPanel extends javax.swing.JPanel implements ExplorerMa ((AutoIngestJobsNode) explorerManager.getRootContext()).refresh(refreshEvent); } else { //Make a new AutoIngestJobsNode with it's own EventBus and set it as the root context - explorerManager.setRootContext(new AutoIngestJobsNode(refreshEvent.getJobsSnapshot(), status, new EventBus("AutoIngestJobsNodeEventBus"))); + explorerManager.setRootContext(new AutoIngestJobsNode(refreshEvent.getMonitor(), status, new EventBus("AutoIngestJobsNodeEventBus"))); } outline.setRowSelectionAllowed(true); outline.setFocusable(true); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 77ddcf8d63..8dddd2a3e6 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -490,6 +490,14 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen break; case RESUME: resume(); + + /** + * Kick off an immediate scan so that the next pending job + * will be picked up sooner than having to wait for the + * InputDirScannerTask to run again. + */ + scanInputDirsNow(); + break; case SHUTDOWN: shutDown(); @@ -1840,13 +1848,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen */ setChanged(); notifyObservers(Event.RESUMED); - - /** - * Publish an event to let remote listeners know that the - * node has been resumed. - */ - eventPublisher.publishRemotely(lastPublishedStateEvent = new AutoIngestNodeStateEvent(Event.RESUMED, AutoIngestManager.LOCAL_HOST_NAME)); } + /** + * Publish an event to let remote listeners know that the node + * has been resumed. + */ + eventPublisher.publishRemotely(lastPublishedStateEvent = new AutoIngestNodeStateEvent(Event.RESUMED, AutoIngestManager.LOCAL_HOST_NAME)); + pauseLock.notifyAll(); } } @@ -2790,6 +2798,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen sysLogger.log(Level.INFO, "Finished ingest modules analysis for {0} ", manifestPath); IngestJob.ProgressSnapshot jobSnapshot = ingestJob.getSnapshot(); for (IngestJob.ProgressSnapshot.DataSourceProcessingSnapshot snapshot : jobSnapshot.getDataSourceSnapshots()) { + AutoIngestJobLogger nestedJobLogger = new AutoIngestJobLogger(manifestPath, snapshot.getDataSource(), caseDirectoryPath); if (!snapshot.isCancelled()) { List cancelledModules = snapshot.getCancelledDataSourceIngestModules(); if (!cancelledModules.isEmpty()) { @@ -2798,15 +2807,15 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen setCaseNodeDataErrorsOccurred(caseDirectoryPath); for (String module : snapshot.getCancelledDataSourceIngestModules()) { sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); - jobLogger.logIngestModuleCancelled(module); + nestedJobLogger.logIngestModuleCancelled(module); } } - jobLogger.logAnalysisCompleted(); + nestedJobLogger.logAnalysisCompleted(); } else { currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); currentJob.setErrorsOccurred(true); setCaseNodeDataErrorsOccurred(caseDirectoryPath); - jobLogger.logAnalysisCancelled(); + nestedJobLogger.logAnalysisCancelled(); CancellationReason cancellationReason = snapshot.getCancellationReason(); if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { throw new AnalysisStartupException(String.format("Analysis cancelled due to %s for %s", cancellationReason.getDisplayName(), manifestPath)); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index d9fb71f56c..876a56c47e 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.nio.file.Path; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -116,11 +117,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } catch (AutopsyEventException ex) { throw new AutoIngestMonitorException("Failed to open auto ingest event channel", ex); //NON-NLS } - coordSvcQueryExecutor.scheduleWithFixedDelay(new CoordinationServiceQueryTask(), 0, CORRD_SVC_QUERY_INERVAL_MINS, TimeUnit.MINUTES); + coordSvcQueryExecutor.scheduleWithFixedDelay(new StateRefreshTask(), 0, CORRD_SVC_QUERY_INERVAL_MINS, TimeUnit.MINUTES); eventPublisher.addSubscriber(EVENT_LIST, this); - // Publish an event that asks running nodes to send their state. - eventPublisher.publishRemotely(new AutoIngestRequestNodeStateEvent(AutoIngestManager.Event.REPORT_STATE)); + refreshNodeState(); } /** @@ -172,7 +172,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removePendingJob(event.getJob()); jobsSnapshot.addOrReplaceRunningJob(event.getJob()); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -190,7 +190,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removePendingJob(job); // Update the state of the existing job in the running jobs table - for (AutoIngestJob runningJob : jobsSnapshot.getRunningJobs()) { + for (AutoIngestJob runningJob : getRunningJobs()) { if (runningJob.equals(job)) { runningJob.setIngestJobsSnapshot(job.getIngestJobSnapshots()); runningJob.setIngestThreadSnapshot(job.getIngestThreadActivitySnapshots()); @@ -200,7 +200,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } } setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -216,7 +216,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removeRunningJob(job); jobsSnapshot.addOrReplaceCompletedJob(job); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -226,7 +226,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @param event A job/case prioritization event. */ private void handleCasePrioritizationEvent(AutoIngestCasePrioritizedEvent event) { - coordSvcQueryExecutor.submit(new CoordinationServiceQueryTask()); + coordSvcQueryExecutor.submit(new StateRefreshTask()); } /** @@ -235,7 +235,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @param event A job/case deletion event. */ private void handleCaseDeletedEvent(AutoIngestCaseDeletedEvent event) { - coordSvcQueryExecutor.submit(new CoordinationServiceQueryTask()); + coordSvcQueryExecutor.submit(new StateRefreshTask()); } /** @@ -259,15 +259,35 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } /** - * Gets the auto ingest monitor's current snapshot of the pending jobs - * queue, running jobs list, and completed jobs list for an auto ingest - * cluster. + * Gets the snapshot of the pending jobs queue for an auto ingest cluster. * - * @return The snapshot. + * @return The pending jobs queue. */ - JobsSnapshot getJobsSnapshot() { + List getPendingJobs() { synchronized (jobsLock) { - return jobsSnapshot; + return new ArrayList<>(jobsSnapshot.pendingJobs); + } + } + + /** + * Gets the snapshot of the running jobs list for an auto ingest cluster. + * + * @return The running jobs list. + */ + List getRunningJobs() { + synchronized (jobsLock) { + return new ArrayList<>(jobsSnapshot.runningJobs); + } + } + + /** + * Gets the snapshot of the completed jobs list for an auto ingest cluster. + * + * @return The completed jobs list. + */ + List getCompletedJobs() { + synchronized (jobsLock) { + return new ArrayList<>(jobsSnapshot.completedJobs); } } @@ -277,7 +297,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @return */ List getNodeStates() { - return nodeStates.values().stream().collect(Collectors.toList()); + // We only report the state for nodes for which we have received + // a 'state' event in the last 15 minutes. + return nodeStates.values() + .stream() + .filter(s -> s.getLastSeenTime().isAfter(Instant.now().minus(Duration.ofMinutes(15)))) + .collect(Collectors.toList()); } /** @@ -287,13 +312,20 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * * @return The refreshed snapshot. */ - JobsSnapshot refreshJobsSnapshot() { + void refreshJobsSnapshot() { synchronized (jobsLock) { jobsSnapshot = queryCoordinationService(); - return jobsSnapshot; } } + /** + * Ask running auto ingest nodes to report their state. + */ + private void refreshNodeState() { + // Publish an event that asks running nodes to send their state. + eventPublisher.publishRemotely(new AutoIngestRequestNodeStateEvent(AutoIngestManager.Event.REPORT_STATE)); + } + /** * Gets a new snapshot of the pending jobs queue, running jobs list, and * completed jobs list for an auto ingest cluster. @@ -358,13 +390,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error removing the * priority of the jobs for the case. * - * @return The latest jobs snapshot. */ - JobsSnapshot deprioritizeCase(final String caseName) throws AutoIngestMonitorException { + void deprioritizeCase(final String caseName) throws AutoIngestMonitorException { List jobsToDeprioritize = new ArrayList<>(); synchronized (jobsLock) { - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getManifest().getCaseName().equals(caseName)) { jobsToDeprioritize.add(pendingJob); } @@ -395,7 +426,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_DEPRIORITIZED, "")); }).start(); } - return jobsSnapshot; } } @@ -407,13 +437,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error bumping the * priority of the jobs for the case. * - * @return The latest jobs snapshot. */ - JobsSnapshot prioritizeCase(final String caseName) throws AutoIngestMonitorException { + void prioritizeCase(final String caseName) throws AutoIngestMonitorException { List jobsToPrioritize = new ArrayList<>(); int highestPriority = 0; synchronized (jobsLock) { - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getPriority() > highestPriority) { highestPriority = pendingJob.getPriority(); } @@ -448,7 +477,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_PRIORITIZED, "")); }).start(); } - return jobsSnapshot; } } @@ -460,15 +488,14 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error removing the * priority of the job. * - * @return The latest jobs snapshot. */ - JobsSnapshot deprioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { + void deprioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { AutoIngestJob jobToDeprioritize = null; /* * Make sure the job is still in the pending jobs queue. */ - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.equals(job)) { jobToDeprioritize = job; break; @@ -506,7 +533,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen }).start(); } - return jobsSnapshot; } } @@ -518,9 +544,8 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error bumping the * priority of the job. * - * @return The latest jobs snapshot. */ - JobsSnapshot prioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { + void prioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { int highestPriority = 0; AutoIngestJob jobToPrioritize = null; @@ -528,7 +553,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * Get the highest known priority and make sure the job is still in * the pending jobs queue. */ - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getPriority() > highestPriority) { highestPriority = pendingJob.getPriority(); } @@ -569,7 +594,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen }).start(); } - return jobsSnapshot; } } @@ -591,7 +615,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen */ void reprocessJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { - if (!jobsSnapshot.getCompletedJobs().contains(job)) { + if (!getCompletedJobs().contains(job)) { return; } @@ -660,7 +684,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen // Update the state of completed jobs associated with this case to indicate // that the case has been deleted - for (AutoIngestJob completedJob : jobsSnapshot.getCompletedJobs()) { + for (AutoIngestJob completedJob : getCompletedJobs()) { if (caseName.equals(completedJob.getManifest().getCaseName())) { try { completedJob.setProcessingStatus(DELETED); @@ -724,25 +748,26 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } /** - * A task that queries the coordination service for auto ingest manifest - * node data and converts it to auto ingest jobs for publication top its - * observers. + * A task that updates the state maintained by the monitor. + * At present this includes auto ingest job and auto ingest node data. + * The job data is refreshed by querying the coordination service for + * auto ingest manifest nodes. + * The auto ingest node data is refreshed by publishing a message asking + * all nodes to report their state. */ - private final class CoordinationServiceQueryTask implements Runnable { + private final class StateRefreshTask implements Runnable { - /** - * Queries the coordination service for auto ingest manifest node data - * and converts it to auto ingest jobs for publication top its - * observers. - */ @Override public void run() { if (!Thread.currentThread().isInterrupted()) { - synchronized (jobsLock) { - jobsSnapshot = queryCoordinationService(); - setChanged(); - notifyObservers(jobsSnapshot); - } + // Query coordination service for jobs data. + refreshJobsSnapshot(); + + // Ask running auto ingest nodes to report their status. + refreshNodeState(); + + setChanged(); + notifyObservers(); } } @@ -752,42 +777,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * A snapshot of the pending jobs queue, running jobs list and completed * jobs list for an auto ingest cluster. */ - static final class JobsSnapshot { + private static final class JobsSnapshot { private final Set pendingJobs = new HashSet<>(); private final Set runningJobs = new HashSet<>(); private final Set completedJobs = new HashSet<>(); - /** - * Gets the snapshot of the pending jobs queue for an auto ingest - * cluster. - * - * @return The pending jobs queue. - */ - List getPendingJobs() { - return new ArrayList<>(this.pendingJobs); - } - - /** - * Gets the snapshot of the running jobs list for an auto ingest - * cluster. - * - * @return The running jobs list. - */ - List getRunningJobs() { - return new ArrayList<>(this.runningJobs); - } - - /** - * Gets the snapshot of the completed jobs list for an auto ingest - * cluster. - * - * @return The completed jobs list. - */ - List getCompletedJobs() { - return new ArrayList<>(this.completedJobs); - } - /** * Adds an auto job to the snapshot of the pending jobs queue for an * auto ingest cluster. If an equivalent job already exists, it is @@ -886,6 +881,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen private final String nodeName; private final State nodeState; + private final Instant lastSeenTime; AutoIngestNodeState(String name, Event event) { nodeName = name; @@ -915,6 +911,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen nodeState = State.UNKNOWN; break; } + lastSeenTime = Instant.now(); } String getName() { @@ -924,6 +921,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen State getState() { return nodeState; } + + Instant getLastSeenTime() { + return lastSeenTime; + } } /** diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java index 2c554bf00f..338bce31c7 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.experimental.autoingest; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; - /** * Class which contains events to identify what should be refreshed in the * AutoIngestJobsNode @@ -31,19 +29,20 @@ class AutoIngestNodeRefreshEvents { */ static class AutoIngestRefreshEvent { - private final JobsSnapshot jobsSnapshot; + private final AutoIngestMonitor monitor; - AutoIngestRefreshEvent(JobsSnapshot jobs) { - this.jobsSnapshot = jobs; + AutoIngestRefreshEvent(AutoIngestMonitor monitor) { + this.monitor = monitor; } /** - * Get the state of the jobs lists when the event was fired. + * Get the monitor which will provide access to the state of + * the jobs. * * @return */ - JobsSnapshot getJobsSnapshot() { - return this.jobsSnapshot; + AutoIngestMonitor getMonitor() { + return this.monitor; } } @@ -56,8 +55,8 @@ class AutoIngestNodeRefreshEvents { /** * Constructs a RefreshChildrenEvent. */ - RefreshChildrenEvent(JobsSnapshot jobs) { - super(jobs); + RefreshChildrenEvent(AutoIngestMonitor monitor) { + super(monitor); } } @@ -72,11 +71,11 @@ class AutoIngestNodeRefreshEvents { /** * Contructs a RefreshCaseEvent * - * @param jobs The current state of the jobs lists. + * @param monitor The monitor that will provide access to the current state of the jobs lists. * @param name The name of the case whose nodes should be refreshed. */ - RefreshCaseEvent(JobsSnapshot jobs, String name) { - super(jobs); + RefreshCaseEvent(AutoIngestMonitor monitor, String name) { + super(monitor); caseName = name; } @@ -103,11 +102,11 @@ class AutoIngestNodeRefreshEvents { /** * Constructs a RefreshJobEvent. * - * @param jobs The curent state of the jobs lists. + * @param monitor The monitor which will provide access to the current state of the jobs lists. * @param job The job which should be refreshed. */ - RefreshJobEvent(JobsSnapshot jobs, AutoIngestJob job) { - super(jobs); + RefreshJobEvent(AutoIngestMonitor monitor, AutoIngestJob job) { + super(monitor); autoIngestJob = job; } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java index 3d57057bff..006dc3e580 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java @@ -145,7 +145,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor.getJobsSnapshot(), getJob()); + return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor, getJob()); } } @@ -184,7 +184,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor.getJobsSnapshot(), getJob()); + return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor, getJob()); } } @@ -225,7 +225,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor.getJobsSnapshot(), getJob().getManifest().getCaseName()); + return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor, getJob().getManifest().getCaseName()); } } @@ -266,7 +266,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor.getJobsSnapshot(), getJob().getManifest().getCaseName()); + return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor, getJob().getManifest().getCaseName()); } } } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index fe86fb9f0b..d6bef92ab8 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -106,10 +106,22 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter try { imageInMemory = IOUtils.toByteArray(inputStream); } catch (IOException ex) { - logger.log(Level.WARNING, "Unable to perform object detection on " + file.getName(), ex); + logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); return IngestModule.ProcessResult.ERROR; } - Mat originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); + Mat originalImage; + try { + originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); + } catch (CvException ex) { + //The image was something which could not be decoded by OpenCv, our isImageThumbnailSupported(file) check above failed us + logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); //NON-NLS + return IngestModule.ProcessResult.ERROR; + } catch (Exception unexpectedException) { + //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented + logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), unexpectedException); + return IngestModule.ProcessResult.ERROR; + } + MatOfRect detectionRectangles = new MatOfRect(); //the rectangles which reprent the coordinates on the image for where objects were detected for (String classifierKey : classifiers.keySet()) { //apply each classifier to the file @@ -117,7 +129,10 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter classifiers.get(classifierKey).detectMultiScale(originalImage, detectionRectangles); } catch (CvException ignored) { //The image was likely an image which we are unable to generate a thumbnail for, and the classifier was likely one where that is not acceptable - logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName())); //NON-NLS + continue; + } catch (Exception unexpectedException) { + //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented + logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " with object id of " + file.getId() +" while trying to apply classifier " + classifierKey, unexpectedException); continue; } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java index 2b3fc1692f..7fbcaac1f0 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java @@ -203,17 +203,20 @@ final class ImageGalleryOptionsPanel extends javax.swing.JPanel { } void store() { - Case openCase; - try { - openCase = Case.getCurrentCaseThrows(); - } catch (NoCurrentCaseException ex) { - Logger.getLogger(ImageGalleryOptionsPanel.class.getName()).log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return; - } ImageGalleryPreferences.setEnabledByDefault(enabledByDefaultBox.isSelected()); ImageGalleryController.getDefault().setListeningEnabled(enabledForCaseBox.isSelected()); - new PerCaseProperties(openCase).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.ENABLED, Boolean.toString(enabledForCaseBox.isSelected())); ImageGalleryPreferences.setGroupCategorizationWarningDisabled(groupCategorizationWarningBox.isSelected()); + + // If a case is open, save the per case setting + try { + Case openCase = Case.getCurrentCaseThrows(); + new PerCaseProperties(openCase).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.ENABLED, Boolean.toString(enabledForCaseBox.isSelected())); + } catch (NoCurrentCaseException ex) { + // It's not an error if there's no case open + } + + + } /** diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index de9c1f6b4f..fb381160f7 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -360,7 +360,7 @@ public class GroupManager { groupConcatClause = " array_to_string(array_agg(obj_id), ',') as object_ids"; } else { - groupConcatClause = "select group_concat(obj_id) as object_ids"; + groupConcatClause = " group_concat(obj_id) as object_ids"; } String query = "select " + groupConcatClause + " , mime_type from tsk_files group by mime_type "; try (SleuthkitCase.CaseDbQuery executeQuery = controller.getSleuthKitCase().executeQuery(query); //NON-NLS diff --git a/InternalPythonModules/android/browserlocation.py b/InternalPythonModules/android/browserlocation.py index 6db230c2e6..a0d73fdd28 100644 --- a/InternalPythonModules/android/browserlocation.py +++ b/InternalPythonModules/android/browserlocation.py @@ -105,7 +105,7 @@ class BrowserLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactTypeName(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactTypeName()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS trackpoint artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/cachelocation.py b/InternalPythonModules/android/cachelocation.py index 019c03c654..53e60b51ee 100644 --- a/InternalPythonModules/android/cachelocation.py +++ b/InternalPythonModules/android/cachelocation.py @@ -138,7 +138,7 @@ class CacheLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS trackpoint artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/calllog.py b/InternalPythonModules/android/calllog.py index 79d0e82479..b20c0d0512 100644 --- a/InternalPythonModules/android/calllog.py +++ b/InternalPythonModules/android/calllog.py @@ -157,7 +157,7 @@ class CallLogAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index call log artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/contact.py b/InternalPythonModules/android/contact.py index 1d556607e0..22a1f797ec 100644 --- a/InternalPythonModules/android/contact.py +++ b/InternalPythonModules/android/contact.py @@ -164,7 +164,7 @@ class ContactAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index contact artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/googlemaplocation.py b/InternalPythonModules/android/googlemaplocation.py index 444c0477ee..712aacc2be 100644 --- a/InternalPythonModules/android/googlemaplocation.py +++ b/InternalPythonModules/android/googlemaplocation.py @@ -115,7 +115,7 @@ class GoogleMapLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS route artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/tangomessage.py b/InternalPythonModules/android/tangomessage.py index bd65738a6e..514bd48ecb 100644 --- a/InternalPythonModules/android/tangomessage.py +++ b/InternalPythonModules/android/tangomessage.py @@ -116,7 +116,7 @@ class TangoMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index Tango message artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/textmessage.py b/InternalPythonModules/android/textmessage.py index a6b36ad549..5854977e8b 100644 --- a/InternalPythonModules/android/textmessage.py +++ b/InternalPythonModules/android/textmessage.py @@ -130,7 +130,7 @@ class TextMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index text message artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/wwfmessage.py b/InternalPythonModules/android/wwfmessage.py index 446ab78b67..6b41ab2677 100644 --- a/InternalPythonModules/android/wwfmessage.py +++ b/InternalPythonModules/android/wwfmessage.py @@ -125,7 +125,7 @@ class WWFMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index WWF message artifact for keyword search.", artifact.getDisplayName()) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java index c06394bf23..a8dbfb6268 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java @@ -43,7 +43,6 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { private final String keywordSearchErrorDialogHeader = org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.dialogErrorHeader"); protected int filesIndexed; private final Map dataSourceMap = new HashMap<>(); - private final List toolTipList = new ArrayList<>(); private List dataSources = new ArrayList<>(); private final DefaultListModel dataSourceListModel = new DefaultListModel<>(); @@ -153,14 +152,12 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { */ synchronized List getDataSourceArray() { List dsList = new ArrayList<>(); - toolTipList.clear(); Collections.sort(this.dataSources, (DataSource ds1, DataSource ds2) -> ds1.getName().compareTo(ds2.getName())); for (DataSource ds : dataSources) { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); dataSourceMap.put(ds.getId(), displayName); - toolTipList.add(dsName); dsList.add(displayName); } return dsList; @@ -175,22 +172,13 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { } /** - * Get dataSourceMap with object id and data source display name. Add the - * data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. * * @return The list of data source name */ Map getDataSourceMap() { return dataSourceMap; } - - /** - * Get a list of tooltip text for data source - * @return A list of tool tips - */ - List getDataSourceToolTipList() { - return toolTipList; - } /** * Get a list of DataSourceListModel diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index 72a1cbab5a..0c7566f52e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -18,11 +18,11 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.awt.*; +import java.awt.Component; +import java.awt.Cursor; +import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; @@ -32,7 +32,6 @@ import java.util.List; import java.util.Set; import java.util.logging.Level; import javax.swing.JCheckBox; -import javax.swing.JList; import javax.swing.JTable; import javax.swing.ListSelectionModel; import javax.swing.event.ListSelectionEvent; @@ -73,22 +72,6 @@ class DropdownListSearchPanel extends AdHocSearchPanel { dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); - dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(getDataSourceToolTipList().get(index)); - } - } - }); } static synchronized DropdownListSearchPanel getDefault() { @@ -683,11 +666,21 @@ class DropdownListSearchPanel extends AdHocSearchPanel { * Set the dataSourceList enabled if the dataSourceCheckBox is selected */ private void setComponentsEnabled() { - boolean enabled = this.dataSourceCheckBox.isSelected(); - this.dataSourceList.setEnabled(enabled); - if (enabled) { - this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + + if (getDataSourceListModel().size() > 1) { + this.dataSourceCheckBox.setEnabled(true); + + boolean enabled = this.dataSourceCheckBox.isSelected(); + this.dataSourceList.setEnabled(enabled); + if (enabled) { + this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + } else { + this.dataSourceList.setSelectedIndices(new int[0]); + } } else { + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceCheckBox.setSelected(false); + this.dataSourceList.setEnabled(false); this.dataSourceList.setSelectedIndices(new int[0]); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java index b859df85ef..bdc80084c7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java @@ -23,8 +23,6 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.FocusEvent; import java.awt.event.FocusListener; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; @@ -32,7 +30,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.logging.Level; -import javax.swing.JList; import javax.swing.JMenuItem; import javax.swing.event.ListSelectionEvent; import org.openide.util.NbBundle; @@ -88,22 +85,6 @@ public class DropdownSingleTermSearchPanel extends AdHocSearchPanel { this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - JList DsList = (JList) evt.getSource(); - int index = DsList.locationToIndex(evt.getPoint()); - if (index > -1) { - DsList.setToolTipText(getDataSourceToolTipList().get(index)); - } - } - }); } /** @@ -390,18 +371,26 @@ public class DropdownSingleTermSearchPanel extends AdHocSearchPanel { } setComponentsEnabled(); firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); - } /** * Set the dataSourceList enabled if the dataSourceCheckBox is selected */ private void setComponentsEnabled() { - boolean enabled = this.dataSourceCheckBox.isSelected(); - this.dataSourceList.setEnabled(enabled); - if (enabled) { - this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + if (getDataSourceListModel().size() > 1) { + this.dataSourceCheckBox.setEnabled(true); + + boolean enabled = this.dataSourceCheckBox.isSelected(); + this.dataSourceList.setEnabled(enabled); + if (enabled) { + this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + } else { + this.dataSourceList.setSelectedIndices(new int[0]); + } } else { + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceCheckBox.setSelected(false); + this.dataSourceList.setEnabled(false); this.dataSourceList.setSelectedIndices(new int[0]); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 83b770bc58..a8abc03b83 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -28,7 +28,6 @@ import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; @@ -92,6 +91,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { private boolean startedSearching = false; private List textExtractors; private StringsTextExtractor stringExtractor; + private TextFileExtractor txtFileExtractor; private final KeywordSearchJobSettings settings; private boolean initialized = false; private long jobId; @@ -244,6 +244,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule { stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions()); + txtFileExtractor = new TextFileExtractor(); + textExtractors = new ArrayList<>(); //order matters, more specific extractors first textExtractors.add(new HtmlTextExtractor()); @@ -343,7 +345,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { textExtractors.clear(); textExtractors = null; stringExtractor = null; - + txtFileExtractor = null; initialized = false; } @@ -568,6 +570,17 @@ public final class KeywordSearchIngestModule implements FileIngestModule { putIngestStatus(jobId, aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); } + if ((wasTextAdded == false) && (aFile.getNameExtension().equalsIgnoreCase("txt"))) { + try { + if (Ingester.getDefault().indexText(txtFileExtractor, aFile, context)) { + putIngestStatus(jobId, aFile.getId(), IngestStatus.TEXT_INGESTED); + wasTextAdded = true; + } + } catch (IngesterException ex) { + logger.log(Level.WARNING, "Unable to index as unicode", ex); + } + } + // if it wasn't supported or had an error, default to strings if (wasTextAdded == false) { extractStringsAndIndex(aFile); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index e5d93a91d4..4c8999ec47 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * + * * Copyright 2012-2014 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -237,8 +237,19 @@ class SearchEngineURLQueryAnalyzer extends Extract { try { //try to decode the url String decoded = URLDecoder.decode(x, "UTF-8"); //NON-NLS return decoded; - } catch (UnsupportedEncodingException uee) { //if it fails, return the encoded string - logger.log(Level.FINE, "Error during URL decoding ", uee); //NON-NLS + } catch (UnsupportedEncodingException exception) { //if it fails, return the encoded string + logger.log(Level.FINE, "Error during URL decoding, returning undecoded value:" + + "\n\tURL: " + url + + "\n\tUndecoded value: " + x + + "\n\tEngine name: " + eng.getEngineName() + + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS + return x; + } catch (IllegalArgumentException exception) { //if it fails, return the encoded string + logger.log(Level.SEVERE, "Illegal argument passed to URL decoding, returning undecoded value:" + + "\n\tURL: " + url + + "\n\tUndecoded value: " + x + + "\n\tEngine name: " + eng.getEngineName() + + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS) return x; } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index a47d33f22a..8b246b05aa 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -86,11 +86,12 @@ class Util { public static String getBaseDomain(String url) { String host = null; + //strip protocol - String cleanUrl = url.replaceFirst("/.*:\\/\\//", ""); + String cleanUrl = url.replaceFirst(".*:\\/\\/", ""); //strip after slashes - String dirToks[] = cleanUrl.split("/\\//"); + String dirToks[] = cleanUrl.split("\\/"); if (dirToks.length > 0) { host = dirToks[0]; } else { @@ -141,7 +142,6 @@ class Util { if (result == null || result.trim().isEmpty()) { return getBaseDomain(value); } - return result; }