From 9d842be3daa04e3e56dd5c00e7c23b0016fbe7d3 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 28 Jun 2018 16:22:54 -0400 Subject: [PATCH 01/55] 3978-UnicodeTextExtractor --- .../KeywordSearchIngestModule.java | 17 +++- .../keywordsearch/UnicodeTextExtractor.java | 78 +++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 83b770bc58..3ed4f76b38 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -28,7 +28,6 @@ import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; @@ -92,6 +91,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { private boolean startedSearching = false; private List textExtractors; private StringsTextExtractor stringExtractor; + private UnicodeTextExtractor unicodeExtractor; private final KeywordSearchJobSettings settings; private boolean initialized = false; private long jobId; @@ -244,6 +244,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule { stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions()); + unicodeExtractor = new UnicodeTextExtractor(); + textExtractors = new ArrayList<>(); //order matters, more specific extractors first textExtractors.add(new HtmlTextExtractor()); @@ -343,7 +345,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { textExtractors.clear(); textExtractors = null; stringExtractor = null; - + unicodeExtractor = null; initialized = false; } @@ -568,6 +570,17 @@ public final class KeywordSearchIngestModule implements FileIngestModule { putIngestStatus(jobId, aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); } + if ((wasTextAdded == false) && (aFile.getNameExtension().equalsIgnoreCase("txt"))) { + try { + if (Ingester.getDefault().indexText(unicodeExtractor, aFile, context)) { + putIngestStatus(jobId, aFile.getId(), IngestStatus.TEXT_INGESTED); + wasTextAdded = true; + } + } catch (IngesterException ex) { + logger.log(Level.WARNING, "Unable to index as unicode", ex); + } + } + // if it wasn't supported or had an error, default to strings if (wasTextAdded == false) { extractStringsAndIndex(aFile); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java new file mode 100644 index 0000000000..a2189b8f4c --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java @@ -0,0 +1,78 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.logging.Level; +import org.apache.tika.parser.txt.CharsetDetector; +import org.apache.tika.parser.txt.CharsetMatch; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.ReadContentInputStream; + +/** + * Extract text from unicode files + */ +final class UnicodeTextExtractor extends ContentTextExtractor { + + static final private Logger logger = Logger.getLogger(UnicodeTextExtractor.class.getName()); + + @Override + boolean isContentTypeSpecific() { + return true; + } + + @Override + boolean isSupported(Content file, String detectedFormat) { + return true; + } + + @Override + public Reader getReader(Content source) throws TextExtractorException { + CharsetDetector detector = new CharsetDetector(); + ReadContentInputStream stream = new ReadContentInputStream(source); + byte[] byteData = new byte[(int) source.getSize()]; + try { + stream.read(byteData, 0, (int) source.getSize()); + detector.setText(byteData); + CharsetMatch match = detector.detect(); + try { + return new StringReader(match.getString()); + + } catch (IOException ex) { + throw new TextExtractorException("Unable to get string from detected text in UnicodeTextExtractor", ex); + } + } catch (ReadContentInputStream.ReadContentInputStreamException ex) { + throw new TextExtractorException("Unable to read text stream in UnicodeTextExtractor", ex); + } + } + + @Override + public boolean isDisabled() { + return false; + } + + @Override + public void logWarning(String msg, Exception ex) { + logger.log(Level.WARNING, msg, ex); + } + +} From 1ce90afe5931e6a4c2c80b9c93909b4fb5228493 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Thu, 28 Jun 2018 15:32:57 -0600 Subject: [PATCH 02/55] added finally block in case anything fails during search the progress bar will still stop --- .../sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java index 356056a1bc..0e94e561b2 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java @@ -264,7 +264,6 @@ public final class CommonFilesPanel extends javax.swing.JPanel { viewers.add(table); progress.setDisplayName(Bundle.CommonFilesPanel_search_done_searchProgressDisplay()); DataResultTopComponent.createInstance(tabTitle, pathText, tableFilterWithDescendantsNode, metadata.size(), viewers); - progress.finish(); } catch (InterruptedException ex) { LOGGER.log(Level.SEVERE, "Interrupted while loading Common Files", ex); MessageNotifyUtil.Message.error(Bundle.CommonFilesPanel_search_done_interupted()); @@ -285,6 +284,8 @@ public final class CommonFilesPanel extends javax.swing.JPanel { errorMessage = Bundle.CommonFilesPanel_search_done_exception(); } MessageNotifyUtil.Message.error(errorMessage); + } finally { + progress.finish(); } } }.execute(); From 0afee2fbf6d8f94f20970c11d8d22b81f06a4227 Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 28 Jun 2018 18:55:15 -0400 Subject: [PATCH 03/55] 3994: Image gallery doesnt work for Postgres - Fixed sql for SQLite. --- .../autopsy/imagegallery/datamodel/grouping/GroupManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index de9c1f6b4f..fb381160f7 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -360,7 +360,7 @@ public class GroupManager { groupConcatClause = " array_to_string(array_agg(obj_id), ',') as object_ids"; } else { - groupConcatClause = "select group_concat(obj_id) as object_ids"; + groupConcatClause = " group_concat(obj_id) as object_ids"; } String query = "select " + groupConcatClause + " , mime_type from tsk_files group by mime_type "; try (SleuthkitCase.CaseDbQuery executeQuery = controller.getSleuthKitCase().executeQuery(query); //NON-NLS From e40eb48182f5a8f55397cda013fa2a4a13d60753 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Thu, 28 Jun 2018 19:06:14 -0600 Subject: [PATCH 04/55] added tests for revised intra case correlation algorithm --- .../commonfilessearch/IntraCaseUtils.java | 28 +++- .../MatchesInAtLeastTwoSources.java | 127 ++++++++++++++++++ 2 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java index 9fddc6b5e1..f81e561b38 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java @@ -104,16 +104,36 @@ class IntraCaseUtils { } void setUp() { - CaseUtils.createAsCurrentCase(this.caseName); + this.createAsCurrentCase(); final ImageDSProcessor imageDSProcessor = new ImageDSProcessor(); - IngestUtils.addDataSource(imageDSProcessor, imagePath1); - IngestUtils.addDataSource(imageDSProcessor, imagePath2); - IngestUtils.addDataSource(imageDSProcessor, imagePath3); + this.addImageOne(imageDSProcessor); + this.addImageTwo(imageDSProcessor); + this.addImageThree(imageDSProcessor); + this.addImageFour(imageDSProcessor); + } + + void addImageFour(final ImageDSProcessor imageDSProcessor) { IngestUtils.addDataSource(imageDSProcessor, imagePath4); } + void addImageThree(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath3); + } + + void addImageTwo(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath2); + } + + void addImageOne(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath1); + } + + void createAsCurrentCase() { + CaseUtils.createAsCurrentCase(this.caseName); + } + Map getDataSourceMap() throws NoCurrentCaseException, TskCoreException, SQLException { return this.dataSourceLoader.getDataSourceMap(); } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java new file mode 100644 index 0000000000..46394edbab --- /dev/null +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java @@ -0,0 +1,127 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.commonfilessearch; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import junit.framework.Test; +import org.netbeans.junit.NbModuleSuite; +import org.netbeans.junit.NbTestCase; +import org.openide.util.Exceptions; +import org.python.icu.impl.Assert; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.ImageDSProcessor; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.commonfilesearch.AllDataSourcesCommonFilesAlgorithm; +import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata; +import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadataBuilder; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.DOC; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.EMPTY; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.IMG; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.SET1; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.SET4; +import org.sleuthkit.autopsy.ingest.IngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleTemplate; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory; +import org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory; +import org.sleuthkit.autopsy.testutils.IngestUtils; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * + * @author bsweeney + */ +public class MatchesInAtLeastTwoSources extends NbTestCase { + + public static Test suite() { + NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(MatchesInAtLeastTwoSources.class). + clusters(".*"). + enableModules(".*"); + return conf.suite(); + } + + private final IntraCaseUtils utils; + + public MatchesInAtLeastTwoSources(String name) { + super(name); + + this.utils = new IntraCaseUtils(this, "MatchesInAtLeastTwoSources"); + } + + @Override + public void setUp() { + this.utils.createAsCurrentCase(); + + final ImageDSProcessor imageDSProcessor = new ImageDSProcessor(); + + this.utils.addImageOne(imageDSProcessor); + this.utils.addImageFour(imageDSProcessor); + + IngestModuleTemplate hashLookupTemplate = IngestUtils.getIngestModuleTemplate(new HashLookupModuleFactory()); + IngestModuleTemplate mimeTypeLookupTemplate = IngestUtils.getIngestModuleTemplate(new FileTypeIdModuleFactory()); + + ArrayList templates = new ArrayList<>(); + templates.add(hashLookupTemplate); + templates.add(mimeTypeLookupTemplate); + + IngestJobSettings ingestJobSettings = new IngestJobSettings(IngestedWithHashAndFileType.class.getCanonicalName(), IngestJobSettings.IngestType.FILES_ONLY, templates); + + try { + IngestUtils.runIngestJob(Case.getCurrentCaseThrows().getDataSources(), ingestJobSettings); + } catch (NoCurrentCaseException | TskCoreException ex) { + Exceptions.printStackTrace(ex); + Assert.fail(ex); + } + } + + @Override + public void tearDown() { + this.utils.tearDown(); + } + + public void testOne() { + try { + Map dataSources = this.utils.getDataSourceMap(); + + CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, false, false); + CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles(); + + Map objectIdToDataSource = IntraCaseUtils.mapFileInstancesToDataSources(metadata); + + List files = IntraCaseUtils.getFiles(objectIdToDataSource.keySet()); + + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET1, 0)); + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET4, 0)); + + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET1, 0)); + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET4, 0)); + + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET1, 0)); + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET4, 0)); + + } catch (NoCurrentCaseException | TskCoreException | SQLException ex) { + Exceptions.printStackTrace(ex); + Assert.fail(ex); + } + } +} From 7168cfe187c61aa76bbdcee6e9ed2be35948c57e Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Thu, 28 Jun 2018 19:08:26 -0600 Subject: [PATCH 05/55] comments --- .../commonfilessearch/MatchesInAtLeastTwoSources.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java index 46394edbab..d45b9a8dc2 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java @@ -48,8 +48,12 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * - * @author bsweeney + * Ensures that matches only are found for files which appear in at least two data sources. + * + * The two datasources used here have no common files. One of the data sources + * has two identical files within it. This should not count as a match. + * + * None of the test files should be found in the results of this test. */ public class MatchesInAtLeastTwoSources extends NbTestCase { From a36c4a61e3ba72d014425e14a9120f86dc5d8558 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Thu, 28 Jun 2018 19:27:52 -0600 Subject: [PATCH 06/55] trying to quiet codacy --- .../MatchesInAtLeastTwoSources.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java index d45b9a8dc2..ba178a59f6 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java @@ -34,11 +34,7 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.commonfilesearch.AllDataSourcesCommonFilesAlgorithm; import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata; import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadataBuilder; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.DOC; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.EMPTY; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.IMG; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.SET1; -import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.SET4; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.*; import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestModuleTemplate; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory; @@ -48,11 +44,12 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * Ensures that matches only are found for files which appear in at least two data sources. - * - * The two datasources used here have no common files. One of the data sources - * has two identical files within it. This should not count as a match. - * + * Ensures that matches only are found for files which appear in at least two + * data sources. + * + * The two datasources used here have no common files. One of the data sources + * has two identical files within it. This should not count as a match. + * * None of the test files should be found in the results of this test. */ public class MatchesInAtLeastTwoSources extends NbTestCase { @@ -109,20 +106,20 @@ public class MatchesInAtLeastTwoSources extends NbTestCase { CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, false, false); CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles(); - + Map objectIdToDataSource = IntraCaseUtils.mapFileInstancesToDataSources(metadata); List files = IntraCaseUtils.getFiles(objectIdToDataSource.keySet()); - + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET1, 0)); assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET4, 0)); - + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET1, 0)); assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET4, 0)); - + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET1, 0)); assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET4, 0)); - + } catch (NoCurrentCaseException | TskCoreException | SQLException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex); From 4693c6681240f6fde5054c9fb1850d3626a505ab Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 28 Jun 2018 22:39:32 -0400 Subject: [PATCH 07/55] Disable controls when fewer than two data sources available. --- .../DropdownListSearchPanel.java | 22 ++++++++++++++----- .../DropdownSingleTermSearchPanel.java | 18 ++++++++++----- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index 72a1cbab5a..e5c4191201 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -18,7 +18,9 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.awt.*; +import java.awt.Component; +import java.awt.Cursor; +import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.MouseEvent; @@ -683,11 +685,21 @@ class DropdownListSearchPanel extends AdHocSearchPanel { * Set the dataSourceList enabled if the dataSourceCheckBox is selected */ private void setComponentsEnabled() { - boolean enabled = this.dataSourceCheckBox.isSelected(); - this.dataSourceList.setEnabled(enabled); - if (enabled) { - this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + + if (getDataSourceListModel().size() > 1) { + this.dataSourceCheckBox.setEnabled(true); + + boolean enabled = this.dataSourceCheckBox.isSelected(); + this.dataSourceList.setEnabled(enabled); + if (enabled) { + this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + } else { + this.dataSourceList.setSelectedIndices(new int[0]); + } } else { + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceCheckBox.setSelected(false); + this.dataSourceList.setEnabled(false); this.dataSourceList.setSelectedIndices(new int[0]); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java index b859df85ef..ba872b0e4d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java @@ -390,18 +390,26 @@ public class DropdownSingleTermSearchPanel extends AdHocSearchPanel { } setComponentsEnabled(); firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); - } /** * Set the dataSourceList enabled if the dataSourceCheckBox is selected */ private void setComponentsEnabled() { - boolean enabled = this.dataSourceCheckBox.isSelected(); - this.dataSourceList.setEnabled(enabled); - if (enabled) { - this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + if (getDataSourceListModel().size() > 1) { + this.dataSourceCheckBox.setEnabled(true); + + boolean enabled = this.dataSourceCheckBox.isSelected(); + this.dataSourceList.setEnabled(enabled); + if (enabled) { + this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + } else { + this.dataSourceList.setSelectedIndices(new int[0]); + } } else { + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceCheckBox.setSelected(false); + this.dataSourceList.setEnabled(false); this.dataSourceList.setSelectedIndices(new int[0]); } } From 5a458e3e7261dfbcd6f104d90d2934603e546c8e Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 08:05:39 -0600 Subject: [PATCH 08/55] more logging --- .../CommonFilesSearchAction.java | 1 - .../CommonFilesSearchResultsViewerTable.java | 43 +++++++++++-------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java index 1727c9322b..5a3f887564 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java @@ -24,7 +24,6 @@ import org.openide.util.HelpCtx; import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.core.Installer; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.coreutils.Logger; diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index 71ffe1a21e..fecb676f4b 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -23,23 +23,27 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.table.TableColumn; import javax.swing.table.TableColumnModel; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; /** - * DataResultViewerTable which overrides the default column - * header width calculations. The CommonFilesSearchResultsViewerTable - * presents multiple tiers of data which are not always present and it may not - * make sense to try to calculate the column widths for such tables by sampling - * rows and looking for wide cells. Rather, we just pick some reasonable values. + * DataResultViewerTable which overrides the default column header + * width calculations. The CommonFilesSearchResultsViewerTable + * presents multiple tiers of data which are not always present and it may not + * make sense to try to calculate the column widths for such tables by sampling + * rows and looking for wide cells. Rather, we just pick some reasonable values. */ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { - + private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; - + + private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); + static { Map map = new HashMap<>(); map.put(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), 260); @@ -49,10 +53,10 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { map.put(Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), 100); map.put(Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), 130); map.put(Bundle.CommonFilesSearchResultsViewerTable_tagsColLbl1(), 300); - + COLUMN_WIDTHS = Collections.unmodifiableMap(map); } - + @NbBundle.Messages({ "CommonFilesSearchResultsViewerTable.filesColLbl=Files", "CommonFilesSearchResultsViewerTable.instancesColLbl=Instances", @@ -63,18 +67,23 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { "CommonFilesSearchResultsViewerTable.tagsColLbl1=Tags" }) @Override - protected void setColumnWidths(){ + protected void setColumnWidths() { TableColumnModel model = this.getColumnModel(); - + Enumeration columnsEnumerator = model.getColumns(); - while(columnsEnumerator.hasMoreElements()){ - + while (columnsEnumerator.hasMoreElements()) { + TableColumn column = columnsEnumerator.nextElement(); - + final String headerValue = column.getHeaderValue().toString(); - final Integer get = COLUMN_WIDTHS.get(headerValue); - - column.setPreferredWidth(get); + + try { + final Integer get = COLUMN_WIDTHS.get(headerValue); + + column.setPreferredWidth(get); + } catch (NullPointerException e) { + LOGGER.log(Level.WARNING, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); + } } } } From f878ca5cd287737b4dc4062c604823645243b98b Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 29 Jun 2018 10:30:24 -0400 Subject: [PATCH 09/55] 3978 add checks for confidence and size to Unicode extractor --- .../keywordsearch/UnicodeTextExtractor.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java index a2189b8f4c..d0159a3f6c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java @@ -29,10 +29,18 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ReadContentInputStream; /** - * Extract text from unicode files + * Extract text from unicode files */ final class UnicodeTextExtractor extends ContentTextExtractor { + //Set an upper limit on the amount of data in a single file to index + static final private int MAX_DATA_SIZE_BYTES = 100000000; + //Set a Minimum confidence value to reject matches that may not have a valid text encoding + //Values of valid text encodings were generally 100, xml code sometimes had a value around 50, + //and pictures and other files with a .txt extention were showing up with a value of 5 or less in limited testing. + //This limited information was used to select the current value as one that would filter out clearly non-text + //files while hopefully working on all files with a valid text encoding + static final private int MIN_MATCH_CONFIDENCE = 20; static final private Logger logger = Logger.getLogger(UnicodeTextExtractor.class.getName()); @Override @@ -49,11 +57,19 @@ final class UnicodeTextExtractor extends ContentTextExtractor { public Reader getReader(Content source) throws TextExtractorException { CharsetDetector detector = new CharsetDetector(); ReadContentInputStream stream = new ReadContentInputStream(source); - byte[] byteData = new byte[(int) source.getSize()]; + int size = (int) source.getSize(); + if (size > MAX_DATA_SIZE_BYTES) { + size = MAX_DATA_SIZE_BYTES; + logger.log(Level.WARNING, "Text file size exceeded 100 mb, ony the first 100 mb has been indexed"); + } + byte[] byteData = new byte[size]; try { - stream.read(byteData, 0, (int) source.getSize()); + stream.read(byteData, 0, size); detector.setText(byteData); CharsetMatch match = detector.detect(); + if (match.getConfidence() < MIN_MATCH_CONFIDENCE) { + throw new TextExtractorException("Text does not match any character set with a high enough confidence for UnicodeTextExtractor"); + } try { return new StringReader(match.getString()); From 7b6ce83acc6cc81027db7f7f96f93a423a8e1a76 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 11:10:50 -0600 Subject: [PATCH 10/55] removed some broken and uneeded code --- .../datamodel/CommonFileChildNodeLoading.java | 95 ------------------- .../datamodel/DisplayableItemNodeVisitor.java | 7 -- 2 files changed, 102 deletions(-) delete mode 100644 Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java b/Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java deleted file mode 100644 index 0c83a12f61..0000000000 --- a/Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * - * Autopsy Forensic Browser - * - * Copyright 2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.datamodel; - -import java.util.LinkedHashMap; -import java.util.Map; -import org.openide.nodes.Children; -import org.openide.nodes.Sheet; -import org.openide.util.NbBundle; - -/** - * A dummy node used by the child factory to display while children are being - * generated - */ -public class CommonFileChildNodeLoading extends DisplayableItemNode { - - public CommonFileChildNodeLoading(Children children) { - super(children); - } - - @Override - public T accept(DisplayableItemNodeVisitor visitor) { - return visitor.visit(this); - } - - @Override - public boolean isLeafTypeNode() { - return true; - } - - @Override - public String getItemType() { - return getClass().getName(); - } - - @Override - protected Sheet createSheet() { - Sheet sheet = new Sheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - } - - Map map = new LinkedHashMap<>(); - map.put(CommonFileChildLoadingPropertyType.File.toString(), "Loading..."); - - final String NO_DESCR = Bundle.AbstractFsContentNode_noDesc_text(); - for (CommonFileChildLoadingPropertyType propType : CommonFileChildLoadingPropertyType.values()) { - final String propString = propType.toString(); - sheetSet.put(new NodeProperty<>(propString, propString, NO_DESCR, map.get(propString))); - } - - return sheet; - } - - /** - * Represents the sole column for the 'dummy' loading node. - */ - @NbBundle.Messages({ - "CommonFileChildLoadingPropertyType.fileColLbl=File" - }) - public enum CommonFileChildLoadingPropertyType { - - File(Bundle.CommonFileChildLoadingPropertyType_fileColLbl()); - - final private String displayString; - - private CommonFileChildLoadingPropertyType(String displayString) { - this.displayString = displayString; - } - - @Override - public String toString() { - return displayString; - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java b/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java index 0e9bd1fcba..9cd13a2f4a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java @@ -120,8 +120,6 @@ public interface DisplayableItemNodeVisitor { T visit(CommonFilesNode cfn); T visit(FileInstanceNode fin); - - T visit(CommonFileChildNodeLoading cfcnl); T visit(InstanceCountNode icn); @@ -213,11 +211,6 @@ public interface DisplayableItemNodeVisitor { return defaultVisit(icn); } - @Override - public T visit(CommonFileChildNodeLoading cfcnl) { - return defaultVisit(cfcnl); - } - @Override public T visit(DirectoryNode dn) { return defaultVisit(dn); From 20f7993114d9f013faa077c73eb5799b7fb5f87a Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 29 Jun 2018 13:34:02 -0400 Subject: [PATCH 11/55] 4003 preserve existing comment when replacing tag --- .../autopsy/actions/ReplaceBlackboardArtifactTagAction.java | 6 +++--- .../sleuthkit/autopsy/actions/ReplaceContentTagAction.java | 4 ++-- .../src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java b/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java index 17fbb40e1b..a2895b4d55 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java @@ -64,14 +64,14 @@ public final class ReplaceBlackboardArtifactTagAction extends ReplaceTagAction() { @Override @@ -91,7 +91,7 @@ public final class ReplaceBlackboardArtifactTagAction extends ReplaceTagAction "# {1} - content obj id", "ReplaceContentTagAction.replaceTag.alert=Unable to replace tag {0} for {1}."}) @Override - protected void replaceTag(ContentTag oldTag, TagName newTagName, String comment) { + protected void replaceTag(ContentTag oldTag, TagName newTagName, String newComment) { new SwingWorker() { @Override @@ -84,7 +84,7 @@ public final class ReplaceContentTagAction extends ReplaceTagAction logger.log(Level.INFO, "Replacing tag {0} with tag {1} for artifact {2}", new Object[]{oldTag.getName().getDisplayName(), newTagName.getDisplayName(), oldTag.getContent().getName()}); //NON-NLS tagsManager.deleteContentTag(oldTag); - tagsManager.addContentTag(oldTag.getContent(), newTagName, comment); + tagsManager.addContentTag(oldTag.getContent(), newTagName, newComment); } catch (TskCoreException tskCoreException) { logger.log(Level.SEVERE, "Error replacing artifact tag", tskCoreException); //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java b/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java index 03d48f4012..79ca4f4739 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java @@ -141,7 +141,7 @@ abstract class ReplaceTagAction extends AbstractAction implements // Add action to replace the tag tagNameItem.addActionListener((ActionEvent event) -> { selectedTags.forEach((oldtag) -> { - replaceTag(oldtag, entry.getValue(), ""); + replaceTag(oldtag, entry.getValue(), oldtag.getComment()); }); }); @@ -178,7 +178,7 @@ abstract class ReplaceTagAction extends AbstractAction implements TagName newTagName = GetTagNameDialog.doDialog(); if (null != newTagName) { selectedTags.forEach((oldtag) -> { - replaceTag(oldtag, newTagName, ""); + replaceTag(oldtag, newTagName, oldtag.getComment()); }); } }); From 6e998d137be05939f8f9f4b2457206b730d18c5c Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 11:38:20 -0600 Subject: [PATCH 12/55] removed debugging code --- .../CommonFilesSearchResultsViewerTable.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index fecb676f4b..e6221c5b50 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -77,13 +77,9 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { final String headerValue = column.getHeaderValue().toString(); - try { - final Integer get = COLUMN_WIDTHS.get(headerValue); + final Integer get = COLUMN_WIDTHS.get(headerValue); - column.setPreferredWidth(get); - } catch (NullPointerException e) { - LOGGER.log(Level.WARNING, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); - } + column.setPreferredWidth(get); } } } From 0d8c299935d8ce6b7260e44396b9eb21d64af3c5 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 11:51:26 -0600 Subject: [PATCH 13/55] removed useless code --- .../commonfilesearch/CommonFilesSearchResultsViewerTable.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index e6221c5b50..db03d08dc6 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -23,8 +23,6 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; import javax.swing.table.TableColumn; import javax.swing.table.TableColumnModel; import org.openide.util.NbBundle; @@ -42,8 +40,6 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; - private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); - static { Map map = new HashMap<>(); map.put(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), 260); From f1ea178b00c2f7c872368a125f0d66af5f15b089 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 12:08:19 -0600 Subject: [PATCH 14/55] handle error conditions gracefully --- .../CommonFilesSearchResultsViewerTable.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index db03d08dc6..9da6a61a21 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -40,6 +40,8 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; + private static final int DEFAULT_WIDTH = 100; + static { Map map = new HashMap<>(); map.put(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), 260); @@ -73,9 +75,16 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { final String headerValue = column.getHeaderValue().toString(); - final Integer get = COLUMN_WIDTHS.get(headerValue); + final Integer defaultWidth = COLUMN_WIDTHS.get(headerValue); - column.setPreferredWidth(get); + if (defaultWidth == null) { + //this should never happen, but if it did, it is because + // we've added a node to the tree that has columns which are + // not defined in the mapping above + column.setPreferredWidth(DEFAULT_WIDTH); + } else { + column.setPreferredWidth(defaultWidth); + } } } } From 9a11e33de055f12b167686ede26e687f358e3495 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 12:27:35 -0600 Subject: [PATCH 15/55] added exception logging back --- .../CommonFilesSearchResultsViewerTable.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index 9da6a61a21..56287dfe32 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -23,6 +23,8 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.table.TableColumn; import javax.swing.table.TableColumnModel; import org.openide.util.NbBundle; @@ -40,7 +42,7 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; - private static final int DEFAULT_WIDTH = 100; + private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); static { Map map = new HashMap<>(); @@ -76,14 +78,11 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { final String headerValue = column.getHeaderValue().toString(); final Integer defaultWidth = COLUMN_WIDTHS.get(headerValue); - - if (defaultWidth == null) { - //this should never happen, but if it did, it is because - // we've added a node to the tree that has columns which are - // not defined in the mapping above - column.setPreferredWidth(DEFAULT_WIDTH); - } else { + + try { column.setPreferredWidth(defaultWidth); + } catch (NullPointerException e) { + LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); } } } From 22af40ab95359d1aee68927886ab614b83d5ed42 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 12:41:13 -0600 Subject: [PATCH 16/55] logging added back --- .../commonfilesearch/CommonFilesSearchResultsViewerTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index 56287dfe32..c005799782 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -81,7 +81,7 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { try { column.setPreferredWidth(defaultWidth); - } catch (NullPointerException e) { + } catch (Exception e) { LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); } } From 1c0643ac3e843b21f801ffe2ad3bf79b9ce93cb2 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Fri, 29 Jun 2018 13:15:23 -0600 Subject: [PATCH 17/55] removed exception handling in favor of null checking --- .../CommonFilesSearchResultsViewerTable.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index c005799782..cacbdd83f9 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -43,6 +43,8 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { private static final long serialVersionUID = 1L; private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); + + private static final int DEFAULT_WIDTH = 100; static { Map map = new HashMap<>(); @@ -79,10 +81,11 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { final Integer defaultWidth = COLUMN_WIDTHS.get(headerValue); - try { + if(defaultWidth == null){ + column.setPreferredWidth(DEFAULT_WIDTH); + LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue)); + } else { column.setPreferredWidth(defaultWidth); - } catch (Exception e) { - LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue), e); } } } From 3b3e0b1a220954b4663de4890d31db53d6b9f5f0 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 29 Jun 2018 15:26:49 -0400 Subject: [PATCH 18/55] 3978 simplify Unicode text extractor code per recommendations from review --- .../keywordsearch/UnicodeTextExtractor.java | 32 ++++++------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java index d0159a3f6c..cbd84c1fe7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.keywordsearch; import java.io.IOException; import java.io.Reader; -import java.io.StringReader; import java.util.logging.Level; import org.apache.tika.parser.txt.CharsetDetector; import org.apache.tika.parser.txt.CharsetMatch; @@ -33,8 +32,6 @@ import org.sleuthkit.datamodel.ReadContentInputStream; */ final class UnicodeTextExtractor extends ContentTextExtractor { - //Set an upper limit on the amount of data in a single file to index - static final private int MAX_DATA_SIZE_BYTES = 100000000; //Set a Minimum confidence value to reject matches that may not have a valid text encoding //Values of valid text encodings were generally 100, xml code sometimes had a value around 50, //and pictures and other files with a .txt extention were showing up with a value of 5 or less in limited testing. @@ -57,28 +54,17 @@ final class UnicodeTextExtractor extends ContentTextExtractor { public Reader getReader(Content source) throws TextExtractorException { CharsetDetector detector = new CharsetDetector(); ReadContentInputStream stream = new ReadContentInputStream(source); - int size = (int) source.getSize(); - if (size > MAX_DATA_SIZE_BYTES) { - size = MAX_DATA_SIZE_BYTES; - logger.log(Level.WARNING, "Text file size exceeded 100 mb, ony the first 100 mb has been indexed"); - } - byte[] byteData = new byte[size]; try { - stream.read(byteData, 0, size); - detector.setText(byteData); - CharsetMatch match = detector.detect(); - if (match.getConfidence() < MIN_MATCH_CONFIDENCE) { - throw new TextExtractorException("Text does not match any character set with a high enough confidence for UnicodeTextExtractor"); - } - try { - return new StringReader(match.getString()); - - } catch (IOException ex) { - throw new TextExtractorException("Unable to get string from detected text in UnicodeTextExtractor", ex); - } - } catch (ReadContentInputStream.ReadContentInputStreamException ex) { - throw new TextExtractorException("Unable to read text stream in UnicodeTextExtractor", ex); + detector.setText(stream); + } catch (IOException ex) { + throw new TextExtractorException("Unable to get string from detected text in UnicodeTextExtractor", ex); } + CharsetMatch match = detector.detect(); + if (match.getConfidence() < MIN_MATCH_CONFIDENCE) { + throw new TextExtractorException("Text does not match any character set with a high enough confidence for UnicodeTextExtractor"); + } + + return match.getReader(); } @Override From 46854d4f7a70080ab1126f3544240aadbd8562a8 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 29 Jun 2018 17:59:42 -0400 Subject: [PATCH 19/55] 3978 rename UnicodeTextExtractor to TextFileExtractor to better reflect its purpose --- .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 8 ++++---- .../{UnicodeTextExtractor.java => TextFileExtractor.java} | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) rename KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/{UnicodeTextExtractor.java => TextFileExtractor.java} (93%) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 3ed4f76b38..a8abc03b83 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -91,7 +91,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { private boolean startedSearching = false; private List textExtractors; private StringsTextExtractor stringExtractor; - private UnicodeTextExtractor unicodeExtractor; + private TextFileExtractor txtFileExtractor; private final KeywordSearchJobSettings settings; private boolean initialized = false; private long jobId; @@ -244,7 +244,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions()); - unicodeExtractor = new UnicodeTextExtractor(); + txtFileExtractor = new TextFileExtractor(); textExtractors = new ArrayList<>(); //order matters, more specific extractors first @@ -345,7 +345,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { textExtractors.clear(); textExtractors = null; stringExtractor = null; - unicodeExtractor = null; + txtFileExtractor = null; initialized = false; } @@ -572,7 +572,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { if ((wasTextAdded == false) && (aFile.getNameExtension().equalsIgnoreCase("txt"))) { try { - if (Ingester.getDefault().indexText(unicodeExtractor, aFile, context)) { + if (Ingester.getDefault().indexText(txtFileExtractor, aFile, context)) { putIngestStatus(jobId, aFile.getId(), IngestStatus.TEXT_INGESTED); wasTextAdded = true; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java similarity index 93% rename from KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java rename to KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java index cbd84c1fe7..bc11515e96 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/UnicodeTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java @@ -28,9 +28,9 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ReadContentInputStream; /** - * Extract text from unicode files + * Extract text from .txt files */ -final class UnicodeTextExtractor extends ContentTextExtractor { +final class TextFileExtractor extends ContentTextExtractor { //Set a Minimum confidence value to reject matches that may not have a valid text encoding //Values of valid text encodings were generally 100, xml code sometimes had a value around 50, @@ -38,7 +38,7 @@ final class UnicodeTextExtractor extends ContentTextExtractor { //This limited information was used to select the current value as one that would filter out clearly non-text //files while hopefully working on all files with a valid text encoding static final private int MIN_MATCH_CONFIDENCE = 20; - static final private Logger logger = Logger.getLogger(UnicodeTextExtractor.class.getName()); + static final private Logger logger = Logger.getLogger(TextFileExtractor.class.getName()); @Override boolean isContentTypeSpecific() { From 4d95be231b599689cca01fd0f41bb96d2b39c3fd Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Mon, 2 Jul 2018 09:12:16 -0400 Subject: [PATCH 20/55] Don't try to add new data sources to the central repo twice. Don't throw an exception when attempting to add an already initialized data source --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 3 ++- .../centralrepository/datamodel/CorrelationDataSource.java | 1 + .../centralrepository/eventlisteners/CaseEventListener.java | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 767e86ebcc..3066340c55 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -432,7 +432,8 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamDataSource.getCaseID() == -1) { throw new EamDbException("Case ID is -1"); } else if (eamDataSource.getID() != -1) { - throw new EamDbException("Database ID is already set in object"); + // This data source is already in the central repo + return; } Connection conn = connect(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java index 8cf07a42e9..84dce834a1 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java @@ -67,6 +67,7 @@ public class CorrelationDataSource implements Serializable { /** * Create a CorrelationDataSource object from a TSK Content object. + * This will add it to the central repository. * * @param correlationCase the current CorrelationCase used for ensuring * uniqueness of DataSource diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 55958287d5..a3d49d6ba3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -450,10 +450,10 @@ final class CaseEventListener implements PropertyChangeListener { correlationCase = dbManager.newCase(openCase); } if (null == dbManager.getDataSource(correlationCase, deviceId)) { - dbManager.newDataSource(CorrelationDataSource.fromTSKDataSource(correlationCase, newDataSource)); + CorrelationDataSource.fromTSKDataSource(correlationCase, newDataSource); } } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS + LOGGER.log(Level.SEVERE, "Error adding new data source to the central repository", ex); //NON-NLS } catch (TskCoreException | TskDataException ex) { LOGGER.log(Level.SEVERE, "Error getting data source from DATA_SOURCE_ADDED event content.", ex); //NON-NLS } From 2d279350693688c45d99b80107f3c488c208aa6d Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 2 Jul 2018 13:07:29 -0400 Subject: [PATCH 21/55] 4006 do not escape cell contents when writing thumbnails --- .../sleuthkit/autopsy/report/ReportHTML.java | 102 ++++++++++-------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index b69d84b629..50dfe318c3 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -556,17 +556,29 @@ class ReportHTML implements TableReportModule { } /** - * Add a row to the current table. + * Add a row to the current table, escaping the text to be contained in the + * row. * * @param row values for each cell in the row */ @Override public void addRow(List row) { + addRow(row, true); + } + + /** + * Add a row to the current table. + * + * @param row values for each cell in the row + * @param escapeText whether or not the text of the row should be escaped, + * true for escaped, false for not escaped + */ + private void addRow(List row, boolean escapeText) { StringBuilder builder = new StringBuilder(); builder.append("\t\n"); //NON-NLS for (String cell : row) { - String escapeHTMLCell = EscapeUtil.escapeHtml(cell); - builder.append("\t\t").append(escapeHTMLCell).append("\n"); //NON-NLS + String cellText = escapeText ? EscapeUtil.escapeHtml(cell) : cell; + builder.append("\t\t").append(cellText).append("\n"); //NON-NLS } builder.append("\t\n"); //NON-NLS rowCount++; @@ -593,7 +605,7 @@ class ReportHTML implements TableReportModule { public void addRowWithTaggedContentHyperlink(List row, ContentTag contentTag) { Content content = contentTag.getContent(); if (content instanceof AbstractFile == false) { - addRow(row); + addRow(row, true); return; } AbstractFile file = (AbstractFile) content; @@ -647,7 +659,7 @@ class ReportHTML implements TableReportModule { int pages = 1; for (Content content : images) { if (currentRow.size() == THUMBNAIL_COLUMNS) { - addRow(currentRow); + addRow(currentRow, false); currentRow.clear(); } @@ -727,7 +739,7 @@ class ReportHTML implements TableReportModule { // Finish out the row. currentRow.add(""); } - addRow(currentRow); + addRow(currentRow, false); } // manually set rowCount to be the total number of images. @@ -1094,8 +1106,8 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS summary.append(writeSummaryCaseDetails()); summary.append(writeSummaryImageInfo()); - summary.append(writeSummarySoftwareInfo(skCase,ingestJobs)); - summary.append(writeSummaryIngestHistoryInfo(skCase,ingestJobs)); + summary.append(writeSummarySoftwareInfo(skCase, ingestJobs)); + summary.append(writeSummaryIngestHistoryInfo(skCase, ingestJobs)); if (generatorLogoSet) { summary.append("
\n"); //NON-NLS summary.append("\n"); //NON-NLS @@ -1126,14 +1138,13 @@ class ReportHTML implements TableReportModule { } } } - + /** * Write the case details section of the summary for this report. - * + * * @return StringBuilder updated html report with case details */ - - private StringBuilder writeSummaryCaseDetails(){ + private StringBuilder writeSummaryCaseDetails() { StringBuilder summary = new StringBuilder(); String caseName = currentCase.getDisplayName(); String caseNumber = currentCase.getNumber(); @@ -1146,40 +1157,39 @@ class ReportHTML implements TableReportModule { imagecount = 0; } summary.append("
\n"); //NON-NLS - if (agencyLogoSet) { - summary.append("
\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - } - final String align = agencyLogoSet ? "right" : "left"; //NON-NLS NON-NLS - summary.append("
\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseName")) //NON-NLS - .append("").append(caseName).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseNum")) //NON-NLS - .append("").append(!caseNumber.isEmpty() ? caseNumber : NbBundle //NON-NLS - .getMessage(this.getClass(), "ReportHTML.writeSum.noCaseNum")).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.examiner")).append("") //NON-NLS - .append(!examiner.isEmpty() ? examiner : NbBundle - .getMessage(this.getClass(), "ReportHTML.writeSum.noExaminer")) - .append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.numImages")) //NON-NLS - .append("").append(imagecount).append("
\n"); //NON-NLS + if (agencyLogoSet) { + summary.append("
\n"); //NON-NLS + summary.append("\n"); //NON-NLS summary.append("
\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - return summary; + } + final String align = agencyLogoSet ? "right" : "left"; //NON-NLS NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseName")) //NON-NLS + .append("").append(caseName).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseNum")) //NON-NLS + .append("").append(!caseNumber.isEmpty() ? caseNumber : NbBundle //NON-NLS + .getMessage(this.getClass(), "ReportHTML.writeSum.noCaseNum")).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.examiner")).append("") //NON-NLS + .append(!examiner.isEmpty() ? examiner : NbBundle + .getMessage(this.getClass(), "ReportHTML.writeSum.noExaminer")) + .append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.numImages")) //NON-NLS + .append("").append(imagecount).append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + return summary; } - + /** * Write the Image Information section of the summary for this report. - * + * * @return StringBuilder updated html report with Image Information */ - private StringBuilder writeSummaryImageInfo() { StringBuilder summary = new StringBuilder(); summary.append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.imageInfoHeading")); @@ -1208,13 +1218,12 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS return summary; } - + /** * Write the software information section of the summary for this report. - * + * * @return StringBuilder updated html report with software information */ - private StringBuilder writeSummarySoftwareInfo(SleuthkitCase skCase, List ingestJobs) { StringBuilder summary = new StringBuilder(); summary.append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.softwareInfoHeading")); @@ -1244,13 +1253,12 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS return summary; } - + /** * Write the Ingest History section of the summary for this report. - * + * * @return StringBuilder updated html report with ingest history */ - private StringBuilder writeSummaryIngestHistoryInfo(SleuthkitCase skCase, List ingestJobs) { StringBuilder summary = new StringBuilder(); try { @@ -1304,4 +1312,4 @@ class ReportHTML implements TableReportModule { + thumbFile.getName(); } -} \ No newline at end of file +} From f4448359029a8d6d16f235eb3f6378fbb53aa13a Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 3 Jul 2018 08:51:14 -0400 Subject: [PATCH 22/55] Add max length for central repo value field --- .../AddEditCentralRepoCommentAction.java | 2 +- .../datamodel/AbstractSqlEamDb.java | 35 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index dfeb4fbdcd..654ee7161e 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -105,7 +105,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { dbManager.updateAttributeInstanceComment(correlationAttribute); } } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); + logger.log(Level.SEVERE, "Error adding comment", ex); } } return centralRepoCommentDialog.getComment(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 767e86ebcc..3d4bd946ea 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -57,7 +57,10 @@ abstract class AbstractSqlEamDb implements EamDb { private int bulkArtifactsCount; protected int bulkArtifactsThreshold; private final Map> bulkArtifacts; - + + // Maximum length for the value column in the instance tables + static final int MAX_VALUE_LENGTH = 128; + // number of instances to keep in bulk queue before doing an insert. // Update Test code if this changes. It's hard coded there. static final int DEFAULT_BULK_THRESHHOLD = 1000; @@ -550,6 +553,12 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamArtifact.getCorrelationValue() == null) { throw new EamDbException("Correlation value is null"); } + + // Don't throw an exception here because the calling code may be looping over values. + if (eamArtifact.getCorrelationValue().length() >= MAX_VALUE_LENGTH) { + logger.log(Level.WARNING, "Artifact value too long for central repository: {0}", eamArtifact.getCorrelationValue()); + return; + } Connection conn = connect(); @@ -984,18 +993,22 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("CorrelationAttributeInstance known status is null"); } - bulkPs.setString(1, eamInstance.getCorrelationCase().getCaseUUID()); - bulkPs.setString(2, eamInstance.getCorrelationDataSource().getDeviceID()); - bulkPs.setInt(3, eamInstance.getCorrelationDataSource().getCaseID()); - bulkPs.setString(4, eamArtifact.getCorrelationValue()); - bulkPs.setString(5, eamInstance.getFilePath()); - bulkPs.setByte(6, eamInstance.getKnownStatus().getFileKnownValue()); - if ("".equals(eamInstance.getComment())) { - bulkPs.setNull(7, Types.INTEGER); + if (eamArtifact.getCorrelationValue().length() < MAX_VALUE_LENGTH) { + bulkPs.setString(1, eamInstance.getCorrelationCase().getCaseUUID()); + bulkPs.setString(2, eamInstance.getCorrelationDataSource().getDeviceID()); + bulkPs.setInt(3, eamInstance.getCorrelationDataSource().getCaseID()); + bulkPs.setString(4, eamArtifact.getCorrelationValue()); + bulkPs.setString(5, eamInstance.getFilePath()); + bulkPs.setByte(6, eamInstance.getKnownStatus().getFileKnownValue()); + if ("".equals(eamInstance.getComment())) { + bulkPs.setNull(7, Types.INTEGER); + } else { + bulkPs.setString(7, eamInstance.getComment()); + } + bulkPs.addBatch(); } else { - bulkPs.setString(7, eamInstance.getComment()); + logger.log(Level.WARNING, "Artifact value too long for central repository: {0}", eamArtifact.getCorrelationValue()); } - bulkPs.addBatch(); } } } From ba6f406add181a7f29ee3a34cf1ef12225169717 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 3 Jul 2018 13:04:14 -0400 Subject: [PATCH 23/55] Add dialog to allow user to switch to grouping view for large cases --- .../sleuthkit/autopsy/casemodule/Case.java | 11 ++ .../autopsy/directorytree/Bundle.properties | 5 + .../DirectoryTreeTopComponent.java | 82 ++++++++++ .../directorytree/GroupDataSourcesDialog.form | 101 ++++++++++++ .../directorytree/GroupDataSourcesDialog.java | 148 ++++++++++++++++++ 5 files changed, 347 insertions(+) create mode 100644 Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form create mode 100644 Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 44bf52503f..a9bbb39907 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -129,6 +129,7 @@ public class Case { private static final String EXPORT_FOLDER = "Export"; //NON-NLS private static final String LOG_FOLDER = "Log"; //NON-NLS private static final String REPORTS_FOLDER = "Reports"; //NON-NLS + private static final String CONFIG_FOLDER = "Config"; // NON-NLS private static final String TEMP_FOLDER = "Temp"; //NON-NLS private static final String MODULE_FOLDER = "ModuleOutput"; //NON-NLS private static final String CASE_ACTION_THREAD_NAME = "%s-case-action"; @@ -1350,6 +1351,16 @@ public class Case { public String getReportDirectory() { return getOrCreateSubdirectory(REPORTS_FOLDER); } + + /** + * Gets the full path to the config directory for this case, creating it if + * it does not exist. + * + * @return The config directory path. + */ + public String getConfigDirectory() { + return getOrCreateSubdirectory(CONFIG_FOLDER); + } /** * Gets the full path to the module output directory for this case, creating diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties b/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties index 0e7f3cf315..926c4b4626 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties @@ -120,3 +120,8 @@ AddExternalViewerRulePanel.exePathTextField.text= AddExternalViewerRulePanel.exePathLabel.text=Path of the program to use for files with this type or extension AddExternalViewerRulePanel.extRadioButton.text=Extension DirectoryTreeTopComponent.groupByDatasourceCheckBox.text=Group by Data Source +GroupDataSourcesDialog.dataSourceCountLabel.text=jLabel1 +GroupDataSourcesDialog.queryLabel.text=Would you like to group by data source for faster loading? +GroupDataSourcesDialog.yesButton.text=Yes +GroupDataSourcesDialog.noButton.text=No +GroupDataSourcesDialog.title=Group by Data Source? \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 7db201f00b..66bf153833 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -24,6 +24,11 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyVetoException; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -35,6 +40,7 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.prefs.PreferenceChangeEvent; import java.util.prefs.PreferenceChangeListener; +import java.util.Properties; import javax.swing.Action; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; @@ -62,6 +68,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataExplorer; import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent; import org.sleuthkit.autopsy.corecomponents.TableFilterNode; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.ModuleSettings; import org.sleuthkit.autopsy.datamodel.ArtifactNodeSelectionInfo; import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode; import org.sleuthkit.autopsy.datamodel.CreditCards; @@ -103,6 +110,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private static final Logger LOGGER = Logger.getLogger(DirectoryTreeTopComponent.class.getName()); private AutopsyTreeChildrenFactory autopsyTreeChildrenFactory; private Children autopsyTreeChildren; + private final long DEFAULT_DATASOURCE_GROUPING_THRESHOLD = 5; // Threshold for prompting the user about grouping by data source + private final String GROUPING_THRESHOLD_NAME = "GroupDataSourceThreshold"; + private final String SETTINGS_FILE = "CasePreferences.properties"; //NON-NLS /** * the constructor @@ -369,7 +379,51 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat public int getPersistenceType() { return TopComponent.PERSISTENCE_NEVER; } + + /** + * Ask the user if they want to group by data source when opening a large case. + * + * @param currentCase + * @param dataSourceCount + */ + private void promptForDataSourceGrouping(Case currentCase, int dataSourceCount) { + Path settingsFile = Paths.get(currentCase.getConfigDirectory(), SETTINGS_FILE); //NON-NLS + if (settingsFile.toFile().exists()) { + // Read the setting + try (InputStream inputStream = Files.newInputStream(settingsFile)) { + Properties props = new Properties(); + props.load(inputStream); + if (props.getProperty("groupByDataSource", "false").equals("true")) { + UserPreferences.setGroupItemsInTreeByDatasource(true); + groupByDatasourceCheckBox.setSelected(true); + } + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Error reading settings file", ex); + } + } else { + GroupDataSourcesDialog dialog = new GroupDataSourcesDialog(dataSourceCount); + dialog.display(); + if (dialog.groupByDataSourceSelected()) { + UserPreferences.setGroupItemsInTreeByDatasource(true); + groupByDatasourceCheckBox.setSelected(true); + } + // Save the response + Properties props = new Properties(); + if(dialog.groupByDataSourceSelected()) { + props.setProperty("groupByDataSource", "true"); + } else { + props.setProperty("groupByDataSource", "false"); + } + + try (OutputStream fos = Files.newOutputStream(settingsFile)) { + props.store(fos, ""); //NON-NLS + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Error writing settings file", ex); + } + } + } + /** * Called only when top component was closed on all workspaces before and * now is opened for the first time on some workspace. The intent is to @@ -377,6 +431,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * existing workspaces. Subclasses will usually perform initializing tasks * here. */ + @NbBundle.Messages({"# {0} - dataSourceCount", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.text=This case contains {0} data sources. Would you like to group by data source for faster loading?", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.title=Group by data source?"}) @Override public void componentOpened() { // change the cursor to "waiting cursor" for this operation @@ -392,6 +449,31 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat if (null == currentCase || currentCase.hasData() == false) { getTree().setRootVisible(false); // hide the root } else { + // If the case contains a lot of data sources, and they aren't already grouping + // by data source, give the user the option to do so before loading the tree. + if (RuntimeProperties.runningWithGUI()) { + long threshold = DEFAULT_DATASOURCE_GROUPING_THRESHOLD; + if (ModuleSettings.settingExists(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME)) { + try { + threshold = Long.parseLong(ModuleSettings.getConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME)); + } catch (NumberFormatException ex) { + LOGGER.log(Level.SEVERE, "Group data sources threshold is not a number", ex); + } + } else { + ModuleSettings.setConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME, String.valueOf(threshold)); + } + + try { + int dataSourceCount = currentCase.getDataSources().size(); + if (! UserPreferences.groupItemsInTreeByDatasource() && + dataSourceCount > threshold) { + promptForDataSourceGrouping(currentCase, dataSourceCount); + } + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error loading data sources", ex); + } + } + // if there's at least one image, load the image and open the top componen autopsyTreeChildrenFactory = new AutopsyTreeChildrenFactory(); autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form new file mode 100644 index 0000000000..e31c97d4ba --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form @@ -0,0 +1,101 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java new file mode 100644 index 0000000000..f21c1efca9 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java @@ -0,0 +1,148 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.directorytree; + +import javax.swing.JFrame; +import org.openide.util.NbBundle; +import org.openide.windows.WindowManager; +import org.sleuthkit.autopsy.core.UserPreferences; + +/** + * + */ +@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives +final class GroupDataSourcesDialog extends javax.swing.JDialog { + + boolean shouldGroupByDataSource = false; + + /** + * Creates new form GroupDataSourcesDialog + */ + @NbBundle.Messages({"# {0} - dataSourceCount", + "GroupDataSourcesDialog.groupDataSources.text=This case contains {0} data sources."}) + GroupDataSourcesDialog(int dataSourceCount) { + super((JFrame) WindowManager.getDefault().getMainWindow()); + initComponents(); + dataSourceCountLabel.setText(Bundle.GroupDataSourcesDialog_groupDataSources_text(dataSourceCount)); + } + + /** + * Display the dialog. + */ + void display() { + setModal(true); + setSize(getPreferredSize()); + setLocationRelativeTo(this.getParent()); + setAlwaysOnTop(false); + pack(); + setVisible(true); + } + + boolean groupByDataSourceSelected() { + return shouldGroupByDataSource; + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + dataSourceCountLabel = new javax.swing.JLabel(); + queryLabel = new javax.swing.JLabel(); + yesButton = new javax.swing.JButton(); + noButton = new javax.swing.JButton(); + + setDefaultCloseOperation(javax.swing.WindowConstants.DISPOSE_ON_CLOSE); + setTitle(org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.title")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(dataSourceCountLabel, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.dataSourceCountLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(queryLabel, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.queryLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(yesButton, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.yesButton.text")); // NOI18N + yesButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + yesButtonActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(noButton, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.noButton.text")); // NOI18N + noButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + noButtonActionPerformed(evt); + } + }); + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); + getContentPane().setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(queryLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(layout.createSequentialGroup() + .addComponent(dataSourceCountLabel) + .addGap(0, 0, Short.MAX_VALUE)) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addGap(0, 0, Short.MAX_VALUE) + .addComponent(yesButton, javax.swing.GroupLayout.PREFERRED_SIZE, 76, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(noButton, javax.swing.GroupLayout.PREFERRED_SIZE, 76, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addContainerGap()) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addComponent(dataSourceCountLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(queryLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(yesButton) + .addComponent(noButton)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + ); + + pack(); + }// //GEN-END:initComponents + + private void yesButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_yesButtonActionPerformed + shouldGroupByDataSource = true; + dispose(); + }//GEN-LAST:event_yesButtonActionPerformed + + private void noButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_noButtonActionPerformed + shouldGroupByDataSource = false; + dispose(); + }//GEN-LAST:event_noButtonActionPerformed + + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JLabel dataSourceCountLabel; + private javax.swing.JButton noButton; + private javax.swing.JLabel queryLabel; + private javax.swing.JButton yesButton; + // End of variables declaration//GEN-END:variables +} From d9578cdfb8270d19a303042dfea75ca6352482c2 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 3 Jul 2018 13:09:03 -0400 Subject: [PATCH 24/55] Throw an exception --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 5 +---- .../eventlisteners/IngestEventsListener.java | 8 ++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 3d4bd946ea..628e6a962a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -553,11 +553,8 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamArtifact.getCorrelationValue() == null) { throw new EamDbException("Correlation value is null"); } - - // Don't throw an exception here because the calling code may be looping over values. if (eamArtifact.getCorrelationValue().length() >= MAX_VALUE_LENGTH) { - logger.log(Level.WARNING, "Artifact value too long for central repository: {0}", eamArtifact.getCorrelationValue()); - return; + throw new EamDbException("Artifact value too long for central repository: " + eamArtifact.getCorrelationValue()); } Connection conn = connect(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 46ebe75541..e3cfad9fe8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -276,12 +276,12 @@ public class IngestEventsListener { } } if (FALSE == eamArtifacts.isEmpty()) { - try { - for (CorrelationAttribute eamArtifact : eamArtifacts) { + for (CorrelationAttribute eamArtifact : eamArtifacts) { + try { dbManager.addArtifact(eamArtifact); + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error adding artifact to database.", ex); //NON-NLS } - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS } } // DATA_ADDED } From 660889d33b4c495342c22100b329404b3f241bbe Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 3 Jul 2018 13:26:00 -0400 Subject: [PATCH 25/55] Codacy --- .../autopsy/directorytree/DirectoryTreeTopComponent.java | 6 +++--- .../autopsy/directorytree/GroupDataSourcesDialog.java | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 66bf153833..97b99103bd 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -110,9 +110,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private static final Logger LOGGER = Logger.getLogger(DirectoryTreeTopComponent.class.getName()); private AutopsyTreeChildrenFactory autopsyTreeChildrenFactory; private Children autopsyTreeChildren; - private final long DEFAULT_DATASOURCE_GROUPING_THRESHOLD = 5; // Threshold for prompting the user about grouping by data source - private final String GROUPING_THRESHOLD_NAME = "GroupDataSourceThreshold"; - private final String SETTINGS_FILE = "CasePreferences.properties"; //NON-NLS + private static final long DEFAULT_DATASOURCE_GROUPING_THRESHOLD = 5; // Threshold for prompting the user about grouping by data source + private static final String GROUPING_THRESHOLD_NAME = "GroupDataSourceThreshold"; + private static final String SETTINGS_FILE = "CasePreferences.properties"; //NON-NLS /** * the constructor diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java index f21c1efca9..7641655d57 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.directorytree; import javax.swing.JFrame; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; -import org.sleuthkit.autopsy.core.UserPreferences; /** * From 842baa2ec3c6a4d3052ae2d84a0dcd3c27e4f12d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 5 Jul 2018 12:44:25 -0400 Subject: [PATCH 26/55] Disable data source search when only one available. --- .../autopsy/filesearch/DataSourcePanel.java | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java index 96d7973d8a..64fec82a92 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java @@ -54,31 +54,42 @@ public class DataSourcePanel extends javax.swing.JPanel { */ public DataSourcePanel() { initComponents(); - this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { - firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); - }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } + if (this.dataSourceList.getModel().getSize() > 1) { + this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { + firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); + }); + this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - @Override - public void mouseMoved(MouseEvent evt) { - if (evt.getSource() instanceof JList) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(toolTipList.get(index)); + @Override + public void mouseDragged(MouseEvent evt) { + //Unused by now + } + + @Override + public void mouseMoved(MouseEvent evt) { + if (evt.getSource() instanceof JList) { + JList dsList = (JList) evt.getSource(); + int index = dsList.locationToIndex(evt.getPoint()); + if (index > -1) { + dsList.setToolTipText(toolTipList.get(index)); + } } } - } - }); + }); + } else { + /* + * Disable data source filtering since there aren't multiple data + * sources to choose from. + */ + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceList.setEnabled(false); + } } /** - * Get dataSourceMap with object id and data source display name. Add the data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. Add the + * data source full name to toolTipList * * @return The list of data source name */ @@ -93,7 +104,7 @@ public class DataSourcePanel extends javax.swing.JPanel { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); - dataSourceMap.put(ds.getId(), displayName); + dataSourceMap.put(ds.getId(), displayName); toolTipList.add(dsName); dsList.add(displayName); } @@ -107,7 +118,8 @@ public class DataSourcePanel extends javax.swing.JPanel { /** * Get a set of data source object ids that are selected. - * @return A set of selected object ids. + * + * @return A set of selected object ids. */ Set getDataSourcesSelected() { Set dataSourceObjIdSet = new HashSet<>(); @@ -124,6 +136,7 @@ public class DataSourcePanel extends javax.swing.JPanel { /** * Is dataSourceCheckBox selected + * * @return true if the dataSoureCheckBox is selected */ boolean isSelected() { @@ -131,7 +144,8 @@ public class DataSourcePanel extends javax.swing.JPanel { } /** - * Enable the dsList and dataSourceNoteLable if the dataSourceCheckBox is checked. + * Enable the dsList and dataSourceNoteLable if the dataSourceCheckBox is + * checked. */ final void setComponentsEnabled() { boolean enabled = this.isSelected(); From 083b8d0bfd94f46e765a01ee304fce75e6486f79 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 5 Jul 2018 15:56:03 -0400 Subject: [PATCH 27/55] 3996 improve opencv logging and prevent any opencv exceptions from escaping process method --- .../ObjectDetectectionFileIngestModule.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index fe86fb9f0b..ca0dd1cf89 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -106,10 +106,22 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter try { imageInMemory = IOUtils.toByteArray(inputStream); } catch (IOException ex) { - logger.log(Level.WARNING, "Unable to perform object detection on " + file.getName(), ex); + logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getName(), ex); return IngestModule.ProcessResult.ERROR; } - Mat originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); + Mat originalImage; + try { + originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); + } catch (CvException ex) { + //The image was something which could not be decoded by OpenCv, our isImageThumbnailSupported(file) check above failed us + logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getName(), ex); //NON-NLS + return IngestModule.ProcessResult.ERROR; + } catch (Exception unexpectedException) { + //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented + logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getName(), unexpectedException); + return IngestModule.ProcessResult.ERROR; + } + MatOfRect detectionRectangles = new MatOfRect(); //the rectangles which reprent the coordinates on the image for where objects were detected for (String classifierKey : classifiers.keySet()) { //apply each classifier to the file @@ -119,6 +131,10 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter //The image was likely an image which we are unable to generate a thumbnail for, and the classifier was likely one where that is not acceptable logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName())); //NON-NLS continue; + } catch (Exception unexpectedException) { + //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented + logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getName() + " while trying to apply classifier " + classifierKey, unexpectedException); + continue; } if (!detectionRectangles.empty()) { From f2e0660829ead6b51b617fcd4d625867ec8115a2 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 5 Jul 2018 16:05:55 -0400 Subject: [PATCH 28/55] 3996 add parent path to logging messages for file for easier finding --- .../objectdetection/ObjectDetectectionFileIngestModule.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index ca0dd1cf89..fd0626041d 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -114,11 +114,11 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); } catch (CvException ex) { //The image was something which could not be decoded by OpenCv, our isImageThumbnailSupported(file) check above failed us - logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getName(), ex); //NON-NLS + logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getParentPath() + file.getName(), ex); //NON-NLS return IngestModule.ProcessResult.ERROR; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented - logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getName(), unexpectedException); + logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getParentPath() + file.getName(), unexpectedException); return IngestModule.ProcessResult.ERROR; } @@ -133,7 +133,7 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter continue; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented - logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getName() + " while trying to apply classifier " + classifierKey, unexpectedException); + logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " while trying to apply classifier " + classifierKey, unexpectedException); continue; } From 3b8b38da6f63d364cc6b268a0425e26f96fe4a65 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 5 Jul 2018 16:12:46 -0400 Subject: [PATCH 29/55] 3996 add object id to log messages for OpenCV related object detection exceptions --- .../ObjectDetectectionFileIngestModule.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index fd0626041d..3e58aac8c1 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -106,7 +106,7 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter try { imageInMemory = IOUtils.toByteArray(inputStream); } catch (IOException ex) { - logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getName(), ex); + logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); return IngestModule.ProcessResult.ERROR; } Mat originalImage; @@ -114,11 +114,11 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); } catch (CvException ex) { //The image was something which could not be decoded by OpenCv, our isImageThumbnailSupported(file) check above failed us - logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getParentPath() + file.getName(), ex); //NON-NLS + logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); //NON-NLS return IngestModule.ProcessResult.ERROR; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented - logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getParentPath() + file.getName(), unexpectedException); + logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), unexpectedException); return IngestModule.ProcessResult.ERROR; } @@ -129,11 +129,11 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter classifiers.get(classifierKey).detectMultiScale(originalImage, detectionRectangles); } catch (CvException ignored) { //The image was likely an image which we are unable to generate a thumbnail for, and the classifier was likely one where that is not acceptable - logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName())); //NON-NLS + logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName() + " with object id of " + file.getId())); //NON-NLS continue; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented - logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " while trying to apply classifier " + classifierKey, unexpectedException); + logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " with object id of " + file.getId() +" while trying to apply classifier " + classifierKey, unexpectedException); continue; } From 5e2f51235871ad4553357c3dda31481d036e38cd Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 6 Jul 2018 11:08:43 -0400 Subject: [PATCH 30/55] 4017 improve logging in regards to when a correlations artifacts value is too long --- .../datamodel/AbstractSqlEamDb.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 9cba07f837..78937a221f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -57,10 +57,10 @@ abstract class AbstractSqlEamDb implements EamDb { private int bulkArtifactsCount; protected int bulkArtifactsThreshold; private final Map> bulkArtifacts; - + // Maximum length for the value column in the instance tables static final int MAX_VALUE_LENGTH = 128; - + // number of instances to keep in bulk queue before doing an insert. // Update Test code if this changes. It's hard coded there. static final int DEFAULT_BULK_THRESHHOLD = 1000; @@ -555,7 +555,11 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation value is null"); } if (eamArtifact.getCorrelationValue().length() >= MAX_VALUE_LENGTH) { - throw new EamDbException("Artifact value too long for central repository: " + eamArtifact.getCorrelationValue()); + throw new EamDbException("Artifact value too long for central repository." + + "\nCorrelationArtifact ID: " + eamArtifact.getID() + + "\nCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\nCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); + } Connection conn = connect(); @@ -982,7 +986,7 @@ abstract class AbstractSqlEamDb implements EamDb { if (!eamArtifact.getCorrelationValue().isEmpty()) { if (eamInstance.getCorrelationCase() == null) { - throw new EamDbException("CorrelationAttributeInstance case is null"); + throw new EamDbException("CorrelationAttributeInstance case is null for :"); } if (eamInstance.getCorrelationDataSource() == null) { throw new EamDbException("CorrelationAttributeInstance data source is null"); @@ -1005,7 +1009,14 @@ abstract class AbstractSqlEamDb implements EamDb { } bulkPs.addBatch(); } else { - logger.log(Level.WARNING, "Artifact value too long for central repository: {0}", eamArtifact.getCorrelationValue()); + logger.log(Level.WARNING, ("Artifact value too long for central repository." + + "\nCorrelationArtifact ID: " + eamArtifact.getID() + + "\nCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\nCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()) + + "\nEam Instance: " + + "\n\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID() + + "\n\tFilePath: " + eamInstance.getFilePath()); } } } @@ -1751,11 +1762,13 @@ abstract class AbstractSqlEamDb implements EamDb { return 0 < badInstances; } + /** * Process the Artifact instance in the EamDb * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance + * * @throws EamDbException */ @Override From c819391112c7d54ec3bc0322603a08d9c2125892 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 6 Jul 2018 11:24:32 -0400 Subject: [PATCH 31/55] 3997 improve logging for recent activity module and cating of illegal arguement exception --- .../recentactivity/SearchEngineURLQueryAnalyzer.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index e5d93a91d4..fdaa47e7ed 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -237,8 +237,13 @@ class SearchEngineURLQueryAnalyzer extends Extract { try { //try to decode the url String decoded = URLDecoder.decode(x, "UTF-8"); //NON-NLS return decoded; - } catch (UnsupportedEncodingException uee) { //if it fails, return the encoded string - logger.log(Level.FINE, "Error during URL decoding ", uee); //NON-NLS + } catch (UnsupportedEncodingException | IllegalArgumentException exception) { //if it fails, return the encoded string + logger.log(Level.FINE, "Error during URL decoding, returning undecoded value:" + + "\nURL: " + url + + "\nUndecoded value: " + x + + "\nEngine name: " + eng.getEngineName() + + "\nEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS + return x; } } From f720d64f09a08c4b43b9b6ee7f752025bc9ce8b9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 6 Jul 2018 11:28:50 -0400 Subject: [PATCH 32/55] Removed tooltips. --- .../autopsy/filesearch/DataSourcePanel.java | 27 ++----------------- .../keywordsearch/AdHocSearchPanel.java | 14 +--------- .../DropdownListSearchPanel.java | 19 ------------- .../DropdownSingleTermSearchPanel.java | 19 ------------- 4 files changed, 3 insertions(+), 76 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java index 96d7973d8a..72877d5054 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.filesearch; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.io.File; import java.util.ArrayList; import java.util.Collections; @@ -29,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; -import javax.swing.JList; import javax.swing.event.ListSelectionEvent; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -47,7 +44,6 @@ public class DataSourcePanel extends javax.swing.JPanel { private static final Logger logger = Logger.getLogger(DataSourcePanel.class.getName()); private static final long serialVersionUID = 1L; private final Map dataSourceMap = new HashMap<>(); - private final List toolTipList = new ArrayList<>(); /** * Creates new form DataSourcePanel @@ -57,28 +53,10 @@ public class DataSourcePanel extends javax.swing.JPanel { this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - if (evt.getSource() instanceof JList) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(toolTipList.get(index)); - } - } - } - }); } /** - * Get dataSourceMap with object id and data source display name. Add the data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. * * @return The list of data source name */ @@ -93,8 +71,7 @@ public class DataSourcePanel extends javax.swing.JPanel { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); - dataSourceMap.put(ds.getId(), displayName); - toolTipList.add(dsName); + dataSourceMap.put(ds.getId(), displayName); dsList.add(displayName); } } catch (NoCurrentCaseException ex) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java index c06394bf23..a8dbfb6268 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java @@ -43,7 +43,6 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { private final String keywordSearchErrorDialogHeader = org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.dialogErrorHeader"); protected int filesIndexed; private final Map dataSourceMap = new HashMap<>(); - private final List toolTipList = new ArrayList<>(); private List dataSources = new ArrayList<>(); private final DefaultListModel dataSourceListModel = new DefaultListModel<>(); @@ -153,14 +152,12 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { */ synchronized List getDataSourceArray() { List dsList = new ArrayList<>(); - toolTipList.clear(); Collections.sort(this.dataSources, (DataSource ds1, DataSource ds2) -> ds1.getName().compareTo(ds2.getName())); for (DataSource ds : dataSources) { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); dataSourceMap.put(ds.getId(), displayName); - toolTipList.add(dsName); dsList.add(displayName); } return dsList; @@ -175,22 +172,13 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { } /** - * Get dataSourceMap with object id and data source display name. Add the - * data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. * * @return The list of data source name */ Map getDataSourceMap() { return dataSourceMap; } - - /** - * Get a list of tooltip text for data source - * @return A list of tool tips - */ - List getDataSourceToolTipList() { - return toolTipList; - } /** * Get a list of DataSourceListModel diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index e5c4191201..0c7566f52e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -23,8 +23,6 @@ import java.awt.Cursor; import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; @@ -34,7 +32,6 @@ import java.util.List; import java.util.Set; import java.util.logging.Level; import javax.swing.JCheckBox; -import javax.swing.JList; import javax.swing.JTable; import javax.swing.ListSelectionModel; import javax.swing.event.ListSelectionEvent; @@ -75,22 +72,6 @@ class DropdownListSearchPanel extends AdHocSearchPanel { dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); - dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(getDataSourceToolTipList().get(index)); - } - } - }); } static synchronized DropdownListSearchPanel getDefault() { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java index ba872b0e4d..bdc80084c7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java @@ -23,8 +23,6 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.FocusEvent; import java.awt.event.FocusListener; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; @@ -32,7 +30,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.logging.Level; -import javax.swing.JList; import javax.swing.JMenuItem; import javax.swing.event.ListSelectionEvent; import org.openide.util.NbBundle; @@ -88,22 +85,6 @@ public class DropdownSingleTermSearchPanel extends AdHocSearchPanel { this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - JList DsList = (JList) evt.getSource(); - int index = DsList.locationToIndex(evt.getPoint()); - if (index > -1) { - DsList.setToolTipText(getDataSourceToolTipList().get(index)); - } - } - }); } /** From 420a3fffe8681cd56518013fea6ee4e65b73bbda Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 6 Jul 2018 12:11:18 -0400 Subject: [PATCH 33/55] 3997 make new logger message SEVERE leave existing one as FINE --- .../datamodel/AbstractSqlEamDb.java | 32 +++++++++++++------ .../SearchEngineURLQueryAnalyzer.java | 26 +++++++++------ 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 78937a221f..b405965b6b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -986,13 +986,25 @@ abstract class AbstractSqlEamDb implements EamDb { if (!eamArtifact.getCorrelationValue().isEmpty()) { if (eamInstance.getCorrelationCase() == null) { - throw new EamDbException("CorrelationAttributeInstance case is null for :"); + throw new EamDbException("CorrelationAttributeInstance case is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); } if (eamInstance.getCorrelationDataSource() == null) { - throw new EamDbException("CorrelationAttributeInstance data source is null"); + throw new EamDbException("CorrelationAttributeInstance data source is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); } if (eamInstance.getKnownStatus() == null) { - throw new EamDbException("CorrelationAttributeInstance known status is null"); + throw new EamDbException("CorrelationAttributeInstance known status is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue() + + "\n\tEam Instance: " + + "\n\t\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\t\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID()); } if (eamArtifact.getCorrelationValue().length() < MAX_VALUE_LENGTH) { @@ -1010,13 +1022,13 @@ abstract class AbstractSqlEamDb implements EamDb { bulkPs.addBatch(); } else { logger.log(Level.WARNING, ("Artifact value too long for central repository." - + "\nCorrelationArtifact ID: " + eamArtifact.getID() - + "\nCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() - + "\nCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()) - + "\nEam Instance: " - + "\n\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() - + "\n\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID() - + "\n\tFilePath: " + eamInstance.getFilePath()); + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()) + + "\n\tEam Instance: " + + "\n\t\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\t\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID() + + "\n\t\tFilePath: " + eamInstance.getFilePath()); } } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index fdaa47e7ed..7bde8113a8 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * + * * Copyright 2012-2014 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -237,13 +237,19 @@ class SearchEngineURLQueryAnalyzer extends Extract { try { //try to decode the url String decoded = URLDecoder.decode(x, "UTF-8"); //NON-NLS return decoded; - } catch (UnsupportedEncodingException | IllegalArgumentException exception) { //if it fails, return the encoded string + } catch (UnsupportedEncodingException exception) { //if it fails, return the encoded string logger.log(Level.FINE, "Error during URL decoding, returning undecoded value:" - + "\nURL: " + url - + "\nUndecoded value: " + x - + "\nEngine name: " + eng.getEngineName() - + "\nEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS - + + "\n\tURL: " + url + + "\n\tUndecoded value: " + x + + "\n\tEngine name: " + eng.getEngineName() + + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS + return x; + } catch (IllegalArgumentException exception) { //if it fails, return the encoded string + logger.log(Level.SEVERE, "Illegal arguement Exception during URL decoding, returning undecoded value:" + + "\n\tURL: " + url + + "\n\tUndecoded value: " + x + + "\n\tEngine name: " + eng.getEngineName() + + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS) return x; } } From 23bc1d632ac8d5952c29524a48940029f76df865 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 6 Jul 2018 16:13:57 -0400 Subject: [PATCH 34/55] Improvement to case log when handling archive data sources --- .../autopsy/experimental/autoingest/AddArchiveTask.java | 3 +-- .../autopsy/experimental/autoingest/AutoIngestManager.java | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index 9830cecbe9..ba554e2309 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -33,7 +33,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.LocalDiskDSProcessor; import org.sleuthkit.autopsy.casemodule.LocalFilesDSProcessor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import static org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; @@ -180,7 +179,7 @@ class AddArchiveTask implements Runnable { synchronized (archiveDspLock) { UUID taskId = UUID.randomUUID(); currentCase.notifyAddingDataSource(taskId); - AutoIngestDataSource internalDataSource = new AutoIngestDataSource(deviceId, newFilePath); + AutoIngestDataSource internalDataSource = new AutoIngestDataSource(newFilePath.getFileName().toString(), newFilePath); DataSourceProcessorCallback internalArchiveDspCallBack = new AddDataSourceCallback(currentCase, internalDataSource, taskId, archiveDspLock); selectedProcessor.process(deviceId, newFilePath, progressMonitor, internalArchiveDspCallBack); archiveDspLock.wait(); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 77ddcf8d63..65b4b5a5ce 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -2790,6 +2790,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen sysLogger.log(Level.INFO, "Finished ingest modules analysis for {0} ", manifestPath); IngestJob.ProgressSnapshot jobSnapshot = ingestJob.getSnapshot(); for (IngestJob.ProgressSnapshot.DataSourceProcessingSnapshot snapshot : jobSnapshot.getDataSourceSnapshots()) { + AutoIngestJobLogger nestedJobLogger = new AutoIngestJobLogger(manifestPath, snapshot.getDataSource(), caseDirectoryPath); if (!snapshot.isCancelled()) { List cancelledModules = snapshot.getCancelledDataSourceIngestModules(); if (!cancelledModules.isEmpty()) { @@ -2798,15 +2799,15 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen setCaseNodeDataErrorsOccurred(caseDirectoryPath); for (String module : snapshot.getCancelledDataSourceIngestModules()) { sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); - jobLogger.logIngestModuleCancelled(module); + nestedJobLogger.logIngestModuleCancelled(module); } } - jobLogger.logAnalysisCompleted(); + nestedJobLogger.logAnalysisCompleted(); } else { currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); currentJob.setErrorsOccurred(true); setCaseNodeDataErrorsOccurred(caseDirectoryPath); - jobLogger.logAnalysisCancelled(); + nestedJobLogger.logAnalysisCancelled(); CancellationReason cancellationReason = snapshot.getCancellationReason(); if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { throw new AnalysisStartupException(String.format("Analysis cancelled due to %s for %s", cancellationReason.getDisplayName(), manifestPath)); From b339a09a2628db52f47e08d8cf7f8156d5c09cde Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 6 Jul 2018 16:19:40 -0400 Subject: [PATCH 35/55] Minor --- .../autopsy/experimental/autoingest/AddArchiveTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index ba554e2309..e62c58343c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -179,7 +179,7 @@ class AddArchiveTask implements Runnable { synchronized (archiveDspLock) { UUID taskId = UUID.randomUUID(); currentCase.notifyAddingDataSource(taskId); - AutoIngestDataSource internalDataSource = new AutoIngestDataSource(newFilePath.getFileName().toString(), newFilePath); + AutoIngestDataSource internalDataSource = new AutoIngestDataSource(deviceId, newFilePath); DataSourceProcessorCallback internalArchiveDspCallBack = new AddDataSourceCallback(currentCase, internalDataSource, taskId, archiveDspLock); selectedProcessor.process(deviceId, newFilePath, progressMonitor, internalArchiveDspCallBack); archiveDspLock.wait(); From fe32cb5500d1d5bc9f9fe20741df3b711acd9d44 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 6 Jul 2018 17:24:45 -0400 Subject: [PATCH 36/55] Update SearchEngineURLQueryAnalyzer.java --- .../autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index 7bde8113a8..4c8999ec47 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -245,7 +245,7 @@ class SearchEngineURLQueryAnalyzer extends Extract { + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS return x; } catch (IllegalArgumentException exception) { //if it fails, return the encoded string - logger.log(Level.SEVERE, "Illegal arguement Exception during URL decoding, returning undecoded value:" + logger.log(Level.SEVERE, "Illegal argument passed to URL decoding, returning undecoded value:" + "\n\tURL: " + url + "\n\tUndecoded value: " + x + "\n\tEngine name: " + eng.getEngineName() From 08aa6ab8a73de925d65c2efa14d6b474c206bb2d Mon Sep 17 00:00:00 2001 From: esaunders Date: Mon, 9 Jul 2018 16:27:49 -0400 Subject: [PATCH 37/55] Publish a resumed message regardless of whether the JobProcessingTask is waiting for directory scan completion or not. --- .../experimental/autoingest/AutoIngestManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 65b4b5a5ce..565c97ca6d 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -1840,13 +1840,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen */ setChanged(); notifyObservers(Event.RESUMED); - - /** - * Publish an event to let remote listeners know that the - * node has been resumed. - */ - eventPublisher.publishRemotely(lastPublishedStateEvent = new AutoIngestNodeStateEvent(Event.RESUMED, AutoIngestManager.LOCAL_HOST_NAME)); } + /** + * Publish an event to let remote listeners know that the node + * has been resumed. + */ + eventPublisher.publishRemotely(lastPublishedStateEvent = new AutoIngestNodeStateEvent(Event.RESUMED, AutoIngestManager.LOCAL_HOST_NAME)); + pauseLock.notifyAll(); } } From aabab0689877e90a4dd33ef6569813f135398459 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 10 Jul 2018 11:57:12 -0400 Subject: [PATCH 38/55] Set UI selections to null before closing top components. --- .../CoreComponentControl.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java index bf26dc8295..06ad27962a 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java @@ -88,7 +88,21 @@ final public class CoreComponentControl { TopComponent directoryTree = null; TopComponent favorites = null; final WindowManager windowManager = WindowManager.getDefault(); + + // Set the UI selections to null before closing the top components. + // Otherwise it may experience errors trying to load data for the closed case. for (Mode mode : windowManager.getModes()) { + for (TopComponent tc : windowManager.getOpenedTopComponents(mode)) { + if(tc instanceof DataContent) { + ((DataContent) tc).setNode(null); + } else if(tc instanceof DataResult) { + ((DataResult) tc).setNode(null); + } + } + } + + for (Mode mode : windowManager.getModes()) { + for (TopComponent tc : windowManager.getOpenedTopComponents(mode)) { String tcName = tc.getName(); @@ -105,7 +119,7 @@ final public class CoreComponentControl { } } } - + if (directoryTree != null) { directoryTree.close(); } From c3e663e66e4f0f027259c9e5bd097efb3c8a7e2a Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 10 Jul 2018 12:50:39 -0400 Subject: [PATCH 39/55] Fix split regex --- .../src/org/sleuthkit/autopsy/recentactivity/Util.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index a47d33f22a..e2ee2248d1 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -86,11 +86,15 @@ class Util { public static String getBaseDomain(String url) { String host = null; + + if(url.contains("file/Documents%20and%20Settings")) { + System.out.println("Long one: " + url); + } //strip protocol String cleanUrl = url.replaceFirst("/.*:\\/\\//", ""); //strip after slashes - String dirToks[] = cleanUrl.split("/\\//"); + String dirToks[] = cleanUrl.split("\\/"); if (dirToks.length > 0) { host = dirToks[0]; } else { @@ -139,9 +143,10 @@ class Util { //was not a valid URL, try a less picky method if (result == null || result.trim().isEmpty()) { + System.out.println("getBaseDomain result: " + getBaseDomain(value)); return getBaseDomain(value); } - + System.out.println("extractDomain result: " + result); return result; } From 72394ee2b59ca9ab0e798460f118b93e7f75f435 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Tue, 10 Jul 2018 15:24:59 -0400 Subject: [PATCH 40/55] Removed excessive logging --- .../objectdetection/ObjectDetectectionFileIngestModule.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index 3e58aac8c1..d6bef92ab8 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -129,7 +129,6 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter classifiers.get(classifierKey).detectMultiScale(originalImage, detectionRectangles); } catch (CvException ignored) { //The image was likely an image which we are unable to generate a thumbnail for, and the classifier was likely one where that is not acceptable - logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName() + " with object id of " + file.getId())); //NON-NLS continue; } catch (Exception unexpectedException) { //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented From f6b6aec267dd00ce2f432562d3633c06dd2bad44 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 11 Jul 2018 11:22:33 -0400 Subject: [PATCH 41/55] Fix protocol regex and increase max value length. --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 2 +- .../src/org/sleuthkit/autopsy/recentactivity/Util.java | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index b405965b6b..f535d58ccd 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -59,7 +59,7 @@ abstract class AbstractSqlEamDb implements EamDb { private final Map> bulkArtifacts; // Maximum length for the value column in the instance tables - static final int MAX_VALUE_LENGTH = 128; + static final int MAX_VALUE_LENGTH = 256; // number of instances to keep in bulk queue before doing an insert. // Update Test code if this changes. It's hard coded there. diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index e2ee2248d1..f7a79b991d 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -87,11 +87,8 @@ class Util { public static String getBaseDomain(String url) { String host = null; - if(url.contains("file/Documents%20and%20Settings")) { - System.out.println("Long one: " + url); - } //strip protocol - String cleanUrl = url.replaceFirst("/.*:\\/\\//", ""); + String cleanUrl = url.replaceFirst(".*:\\/\\/", ""); //strip after slashes String dirToks[] = cleanUrl.split("\\/"); From a72527a4d512fd3c037ccdd8f66be38ef1f2e131 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 11 Jul 2018 11:23:46 -0400 Subject: [PATCH 42/55] Cleanup --- .../src/org/sleuthkit/autopsy/recentactivity/Util.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index f7a79b991d..8b246b05aa 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -140,10 +140,8 @@ class Util { //was not a valid URL, try a less picky method if (result == null || result.trim().isEmpty()) { - System.out.println("getBaseDomain result: " + getBaseDomain(value)); return getBaseDomain(value); } - System.out.println("extractDomain result: " + result); return result; } From 94bdd8684021acc0e8e0ebf52fa98634d77a3873 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 11 Jul 2018 11:35:26 -0400 Subject: [PATCH 43/55] Don't bail out of saving settings if there is no case open --- .../ImageGalleryOptionsPanel.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java index 2b3fc1692f..7fbcaac1f0 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java @@ -203,17 +203,20 @@ final class ImageGalleryOptionsPanel extends javax.swing.JPanel { } void store() { - Case openCase; - try { - openCase = Case.getCurrentCaseThrows(); - } catch (NoCurrentCaseException ex) { - Logger.getLogger(ImageGalleryOptionsPanel.class.getName()).log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return; - } ImageGalleryPreferences.setEnabledByDefault(enabledByDefaultBox.isSelected()); ImageGalleryController.getDefault().setListeningEnabled(enabledForCaseBox.isSelected()); - new PerCaseProperties(openCase).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.ENABLED, Boolean.toString(enabledForCaseBox.isSelected())); ImageGalleryPreferences.setGroupCategorizationWarningDisabled(groupCategorizationWarningBox.isSelected()); + + // If a case is open, save the per case setting + try { + Case openCase = Case.getCurrentCaseThrows(); + new PerCaseProperties(openCase).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.ENABLED, Boolean.toString(enabledForCaseBox.isSelected())); + } catch (NoCurrentCaseException ex) { + // It's not an error if there's no case open + } + + + } /** From 953b3628e19a7cdd83c4b0bcc33ad9ee818d43c9 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 11 Jul 2018 12:14:07 -0400 Subject: [PATCH 44/55] Kick off an immediate scan on resume. --- .../experimental/autoingest/AutoIngestManager.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 565c97ca6d..8dddd2a3e6 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -490,6 +490,14 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen break; case RESUME: resume(); + + /** + * Kick off an immediate scan so that the next pending job + * will be picked up sooner than having to wait for the + * InputDirScannerTask to run again. + */ + scanInputDirsNow(); + break; case SHUTDOWN: shutDown(); From 50e13eb31e997862fe01d188f987361e56766f60 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 11 Jul 2018 14:16:16 -0400 Subject: [PATCH 45/55] Fix for threading issue related to the fact that returning copies of the pending, running and completed lists iterated over the collections without syncrhonization leading to the possibilty that the collections might be modified by another thread at the same time. --- .../autoingest/AutoIngestAdminActions.java | 2 +- .../autoingest/AutoIngestDashboard.java | 9 +- .../autoingest/AutoIngestJobsNode.java | 23 ++-- .../autoingest/AutoIngestJobsPanel.java | 2 +- .../autoingest/AutoIngestMonitor.java | 105 +++++++----------- .../AutoIngestNodeRefreshEvents.java | 31 +++--- .../autoingest/PrioritizationAction.java | 8 +- 7 files changed, 79 insertions(+), 101 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java index 87c805fa12..1e1b82d8e7 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java @@ -338,7 +338,7 @@ final class AutoIngestAdminActions { dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); AutoIngestManager.CaseDeletionResult result = dashboard.getMonitor().deleteCase(job); - dashboard.getCompletedJobsPanel().refresh(new AutoIngestNodeRefreshEvents.RefreshChildrenEvent(dashboard.getMonitor().getJobsSnapshot())); + dashboard.getCompletedJobsPanel().refresh(new AutoIngestNodeRefreshEvents.RefreshChildrenEvent(dashboard.getMonitor())); dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); if (AutoIngestManager.CaseDeletionResult.FAILED == result) { JOptionPane.showMessageDialog(dashboard, diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java index 49e9cdbd17..031aa64daf 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java @@ -38,7 +38,6 @@ import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.core.ServicesMonitor; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeRefreshEvents.RefreshChildrenEvent; /** @@ -257,7 +256,7 @@ final class AutoIngestDashboard extends JPanel implements Observer { @Override public void update(Observable observable, Object arg) { - if (arg instanceof JobsSnapshot) { + if (arg == null ) { EventQueue.invokeLater(() -> { refreshTables(); }); @@ -271,9 +270,9 @@ final class AutoIngestDashboard extends JPanel implements Observer { * @param nodeStateSnapshot The jobs snapshot. */ void refreshTables() { - pendingJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); - runningJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); - completedJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); + pendingJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); + runningJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); + completedJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); } /** diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java index a7b2ad6116..552e44834c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java @@ -33,7 +33,6 @@ import org.openide.nodes.Node; import org.openide.nodes.Sheet; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.datamodel.NodeProperty; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; import org.sleuthkit.autopsy.guiutils.DurationCellRenderer; import org.sleuthkit.autopsy.guiutils.StatusIconCellRenderer; @@ -61,13 +60,13 @@ final class AutoIngestJobsNode extends AbstractNode { /** * Construct a new AutoIngestJobsNode. * - * @param snapshot the snapshot which contains the AutoIngestJobs + * @param monitor the monitor which gives access to the AutoIngestJobs * @param status the status of the jobs being displayed * @param eventBus the event bus which will be used to send and receive * refresh events */ - AutoIngestJobsNode(JobsSnapshot jobsSnapshot, AutoIngestJobStatus status, EventBus eventBus) { - super(Children.create(new AutoIngestNodeChildren(jobsSnapshot, status, eventBus), false)); + AutoIngestJobsNode(AutoIngestMonitor monitor, AutoIngestJobStatus status, EventBus eventBus) { + super(Children.create(new AutoIngestNodeChildren(monitor, status, eventBus), false)); refreshChildrenEventBus = eventBus; } @@ -84,7 +83,7 @@ final class AutoIngestJobsNode extends AbstractNode { static final class AutoIngestNodeChildren extends ChildFactory { private final AutoIngestJobStatus autoIngestJobStatus; - private JobsSnapshot jobsSnapshot; + private AutoIngestMonitor monitor; private final RefreshChildrenSubscriber refreshChildrenSubscriber = new RefreshChildrenSubscriber(); private final EventBus refreshEventBus; @@ -92,13 +91,13 @@ final class AutoIngestJobsNode extends AbstractNode { * Create children nodes for the AutoIngestJobsNode which will each * represent a single AutoIngestJob * - * @param snapshot the snapshot which contains the AutoIngestJobs + * @param monitor the monitor which gives access to the AutoIngestJobs * @param status the status of the jobs being displayed * @param eventBus the event bus which the class registers to for * refresh events */ - AutoIngestNodeChildren(JobsSnapshot snapshot, AutoIngestJobStatus status, EventBus eventBus) { - jobsSnapshot = snapshot; + AutoIngestNodeChildren(AutoIngestMonitor monitor, AutoIngestJobStatus status, EventBus eventBus) { + this.monitor = monitor; autoIngestJobStatus = status; refreshEventBus = eventBus; refreshChildrenSubscriber.register(refreshEventBus); @@ -109,14 +108,14 @@ final class AutoIngestJobsNode extends AbstractNode { List jobs; switch (autoIngestJobStatus) { case PENDING_JOB: - jobs = jobsSnapshot.getPendingJobs(); + jobs = monitor.getPendingJobs(); Collections.sort(jobs); break; case RUNNING_JOB: - jobs = jobsSnapshot.getRunningJobs(); + jobs = monitor.getRunningJobs(); break; case COMPLETED_JOB: - jobs = jobsSnapshot.getCompletedJobs(); + jobs = monitor.getCompletedJobs(); break; default: jobs = new ArrayList<>(); @@ -167,7 +166,7 @@ final class AutoIngestJobsNode extends AbstractNode { //Ignore netbeans suggesting this isn't being used, it is used behind the scenes by the EventBus //RefreshChildrenEvents can change which children are present however //RefreshJobEvents and RefreshCaseEvents can still change the order we want to display them in - jobsSnapshot = refreshEvent.getJobsSnapshot(); + monitor = refreshEvent.getMonitor(); refresh(true); } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java index a3cdc1c51d..fb6f79ab0c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java @@ -171,7 +171,7 @@ final class AutoIngestJobsPanel extends javax.swing.JPanel implements ExplorerMa ((AutoIngestJobsNode) explorerManager.getRootContext()).refresh(refreshEvent); } else { //Make a new AutoIngestJobsNode with it's own EventBus and set it as the root context - explorerManager.setRootContext(new AutoIngestJobsNode(refreshEvent.getJobsSnapshot(), status, new EventBus("AutoIngestJobsNodeEventBus"))); + explorerManager.setRootContext(new AutoIngestJobsNode(refreshEvent.getMonitor(), status, new EventBus("AutoIngestJobsNodeEventBus"))); } outline.setRowSelectionAllowed(true); outline.setFocusable(true); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index d9fb71f56c..1b1b1bd613 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -172,7 +172,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removePendingJob(event.getJob()); jobsSnapshot.addOrReplaceRunningJob(event.getJob()); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -190,7 +190,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removePendingJob(job); // Update the state of the existing job in the running jobs table - for (AutoIngestJob runningJob : jobsSnapshot.getRunningJobs()) { + for (AutoIngestJob runningJob : getRunningJobs()) { if (runningJob.equals(job)) { runningJob.setIngestJobsSnapshot(job.getIngestJobSnapshots()); runningJob.setIngestThreadSnapshot(job.getIngestThreadActivitySnapshots()); @@ -200,7 +200,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } } setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -216,7 +216,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removeRunningJob(job); jobsSnapshot.addOrReplaceCompletedJob(job); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -259,15 +259,35 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } /** - * Gets the auto ingest monitor's current snapshot of the pending jobs - * queue, running jobs list, and completed jobs list for an auto ingest - * cluster. + * Gets the snapshot of the pending jobs queue for an auto ingest cluster. * - * @return The snapshot. + * @return The pending jobs queue. */ - JobsSnapshot getJobsSnapshot() { + List getPendingJobs() { synchronized (jobsLock) { - return jobsSnapshot; + return new ArrayList<>(jobsSnapshot.pendingJobs); + } + } + + /** + * Gets the snapshot of the running jobs list for an auto ingest cluster. + * + * @return The running jobs list. + */ + List getRunningJobs() { + synchronized (jobsLock) { + return new ArrayList<>(jobsSnapshot.runningJobs); + } + } + + /** + * Gets the snapshot of the completed jobs list for an auto ingest cluster. + * + * @return The completed jobs list. + */ + List getCompletedJobs() { + synchronized (jobsLock) { + return new ArrayList<>(jobsSnapshot.completedJobs); } } @@ -287,10 +307,9 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * * @return The refreshed snapshot. */ - JobsSnapshot refreshJobsSnapshot() { + void refreshJobsSnapshot() { synchronized (jobsLock) { jobsSnapshot = queryCoordinationService(); - return jobsSnapshot; } } @@ -358,13 +377,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error removing the * priority of the jobs for the case. * - * @return The latest jobs snapshot. */ - JobsSnapshot deprioritizeCase(final String caseName) throws AutoIngestMonitorException { + void deprioritizeCase(final String caseName) throws AutoIngestMonitorException { List jobsToDeprioritize = new ArrayList<>(); synchronized (jobsLock) { - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getManifest().getCaseName().equals(caseName)) { jobsToDeprioritize.add(pendingJob); } @@ -395,7 +413,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_DEPRIORITIZED, "")); }).start(); } - return jobsSnapshot; } } @@ -407,13 +424,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error bumping the * priority of the jobs for the case. * - * @return The latest jobs snapshot. */ - JobsSnapshot prioritizeCase(final String caseName) throws AutoIngestMonitorException { + void prioritizeCase(final String caseName) throws AutoIngestMonitorException { List jobsToPrioritize = new ArrayList<>(); int highestPriority = 0; synchronized (jobsLock) { - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getPriority() > highestPriority) { highestPriority = pendingJob.getPriority(); } @@ -448,7 +464,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_PRIORITIZED, "")); }).start(); } - return jobsSnapshot; } } @@ -460,15 +475,14 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error removing the * priority of the job. * - * @return The latest jobs snapshot. */ - JobsSnapshot deprioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { + void deprioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { AutoIngestJob jobToDeprioritize = null; /* * Make sure the job is still in the pending jobs queue. */ - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.equals(job)) { jobToDeprioritize = job; break; @@ -506,7 +520,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen }).start(); } - return jobsSnapshot; } } @@ -518,9 +531,8 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error bumping the * priority of the job. * - * @return The latest jobs snapshot. */ - JobsSnapshot prioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { + void prioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { int highestPriority = 0; AutoIngestJob jobToPrioritize = null; @@ -528,7 +540,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * Get the highest known priority and make sure the job is still in * the pending jobs queue. */ - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getPriority() > highestPriority) { highestPriority = pendingJob.getPriority(); } @@ -569,7 +581,6 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen }).start(); } - return jobsSnapshot; } } @@ -591,7 +602,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen */ void reprocessJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { - if (!jobsSnapshot.getCompletedJobs().contains(job)) { + if (!getCompletedJobs().contains(job)) { return; } @@ -660,7 +671,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen // Update the state of completed jobs associated with this case to indicate // that the case has been deleted - for (AutoIngestJob completedJob : jobsSnapshot.getCompletedJobs()) { + for (AutoIngestJob completedJob : getCompletedJobs()) { if (caseName.equals(completedJob.getManifest().getCaseName())) { try { completedJob.setProcessingStatus(DELETED); @@ -741,7 +752,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen synchronized (jobsLock) { jobsSnapshot = queryCoordinationService(); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } } @@ -752,42 +763,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * A snapshot of the pending jobs queue, running jobs list and completed * jobs list for an auto ingest cluster. */ - static final class JobsSnapshot { + private static final class JobsSnapshot { private final Set pendingJobs = new HashSet<>(); private final Set runningJobs = new HashSet<>(); private final Set completedJobs = new HashSet<>(); - /** - * Gets the snapshot of the pending jobs queue for an auto ingest - * cluster. - * - * @return The pending jobs queue. - */ - List getPendingJobs() { - return new ArrayList<>(this.pendingJobs); - } - - /** - * Gets the snapshot of the running jobs list for an auto ingest - * cluster. - * - * @return The running jobs list. - */ - List getRunningJobs() { - return new ArrayList<>(this.runningJobs); - } - - /** - * Gets the snapshot of the completed jobs list for an auto ingest - * cluster. - * - * @return The completed jobs list. - */ - List getCompletedJobs() { - return new ArrayList<>(this.completedJobs); - } - /** * Adds an auto job to the snapshot of the pending jobs queue for an * auto ingest cluster. If an equivalent job already exists, it is diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java index 2c554bf00f..338bce31c7 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.experimental.autoingest; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; - /** * Class which contains events to identify what should be refreshed in the * AutoIngestJobsNode @@ -31,19 +29,20 @@ class AutoIngestNodeRefreshEvents { */ static class AutoIngestRefreshEvent { - private final JobsSnapshot jobsSnapshot; + private final AutoIngestMonitor monitor; - AutoIngestRefreshEvent(JobsSnapshot jobs) { - this.jobsSnapshot = jobs; + AutoIngestRefreshEvent(AutoIngestMonitor monitor) { + this.monitor = monitor; } /** - * Get the state of the jobs lists when the event was fired. + * Get the monitor which will provide access to the state of + * the jobs. * * @return */ - JobsSnapshot getJobsSnapshot() { - return this.jobsSnapshot; + AutoIngestMonitor getMonitor() { + return this.monitor; } } @@ -56,8 +55,8 @@ class AutoIngestNodeRefreshEvents { /** * Constructs a RefreshChildrenEvent. */ - RefreshChildrenEvent(JobsSnapshot jobs) { - super(jobs); + RefreshChildrenEvent(AutoIngestMonitor monitor) { + super(monitor); } } @@ -72,11 +71,11 @@ class AutoIngestNodeRefreshEvents { /** * Contructs a RefreshCaseEvent * - * @param jobs The current state of the jobs lists. + * @param monitor The monitor that will provide access to the current state of the jobs lists. * @param name The name of the case whose nodes should be refreshed. */ - RefreshCaseEvent(JobsSnapshot jobs, String name) { - super(jobs); + RefreshCaseEvent(AutoIngestMonitor monitor, String name) { + super(monitor); caseName = name; } @@ -103,11 +102,11 @@ class AutoIngestNodeRefreshEvents { /** * Constructs a RefreshJobEvent. * - * @param jobs The curent state of the jobs lists. + * @param monitor The monitor which will provide access to the current state of the jobs lists. * @param job The job which should be refreshed. */ - RefreshJobEvent(JobsSnapshot jobs, AutoIngestJob job) { - super(jobs); + RefreshJobEvent(AutoIngestMonitor monitor, AutoIngestJob job) { + super(monitor); autoIngestJob = job; } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java index 3d57057bff..006dc3e580 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java @@ -145,7 +145,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor.getJobsSnapshot(), getJob()); + return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor, getJob()); } } @@ -184,7 +184,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor.getJobsSnapshot(), getJob()); + return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor, getJob()); } } @@ -225,7 +225,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor.getJobsSnapshot(), getJob().getManifest().getCaseName()); + return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor, getJob().getManifest().getCaseName()); } } @@ -266,7 +266,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor.getJobsSnapshot(), getJob().getManifest().getCaseName()); + return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor, getJob().getManifest().getCaseName()); } } } From 3f89863a29d1b44e6273dcfbc4e5a454f62ff559 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 12 Jul 2018 00:40:07 -0400 Subject: [PATCH 46/55] Fix string format of error message --- InternalPythonModules/android/browserlocation.py | 2 +- InternalPythonModules/android/cachelocation.py | 2 +- InternalPythonModules/android/calllog.py | 2 +- InternalPythonModules/android/contact.py | 2 +- InternalPythonModules/android/googlemaplocation.py | 2 +- InternalPythonModules/android/tangomessage.py | 2 +- InternalPythonModules/android/textmessage.py | 2 +- InternalPythonModules/android/wwfmessage.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/InternalPythonModules/android/browserlocation.py b/InternalPythonModules/android/browserlocation.py index 6db230c2e6..a0d73fdd28 100644 --- a/InternalPythonModules/android/browserlocation.py +++ b/InternalPythonModules/android/browserlocation.py @@ -105,7 +105,7 @@ class BrowserLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactTypeName(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactTypeName()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS trackpoint artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/cachelocation.py b/InternalPythonModules/android/cachelocation.py index 019c03c654..53e60b51ee 100644 --- a/InternalPythonModules/android/cachelocation.py +++ b/InternalPythonModules/android/cachelocation.py @@ -138,7 +138,7 @@ class CacheLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS trackpoint artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/calllog.py b/InternalPythonModules/android/calllog.py index 79d0e82479..b20c0d0512 100644 --- a/InternalPythonModules/android/calllog.py +++ b/InternalPythonModules/android/calllog.py @@ -157,7 +157,7 @@ class CallLogAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index call log artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/contact.py b/InternalPythonModules/android/contact.py index 1d556607e0..22a1f797ec 100644 --- a/InternalPythonModules/android/contact.py +++ b/InternalPythonModules/android/contact.py @@ -164,7 +164,7 @@ class ContactAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index contact artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/googlemaplocation.py b/InternalPythonModules/android/googlemaplocation.py index 444c0477ee..712aacc2be 100644 --- a/InternalPythonModules/android/googlemaplocation.py +++ b/InternalPythonModules/android/googlemaplocation.py @@ -115,7 +115,7 @@ class GoogleMapLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS route artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/tangomessage.py b/InternalPythonModules/android/tangomessage.py index bd65738a6e..514bd48ecb 100644 --- a/InternalPythonModules/android/tangomessage.py +++ b/InternalPythonModules/android/tangomessage.py @@ -116,7 +116,7 @@ class TangoMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index Tango message artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/textmessage.py b/InternalPythonModules/android/textmessage.py index a6b36ad549..5854977e8b 100644 --- a/InternalPythonModules/android/textmessage.py +++ b/InternalPythonModules/android/textmessage.py @@ -130,7 +130,7 @@ class TextMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index text message artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/wwfmessage.py b/InternalPythonModules/android/wwfmessage.py index 446ab78b67..6b41ab2677 100644 --- a/InternalPythonModules/android/wwfmessage.py +++ b/InternalPythonModules/android/wwfmessage.py @@ -125,7 +125,7 @@ class WWFMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index WWF message artifact for keyword search.", artifact.getDisplayName()) From 677da001960a0f7638cfa6ab58268f2fc6e05a13 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 12 Jul 2018 13:20:20 -0400 Subject: [PATCH 47/55] Removed message. --- .../org/sleuthkit/autopsy/casemodule/services/Bundle.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties b/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties index e9c1b5307c..266db3a64c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties @@ -1,6 +1,5 @@ OptionsCategory_Name_TagNamesOptions=Tags OptionsCategory_TagNames=TagNames -Blackboard.unableToIndexArtifact.error.msg=Unable to index blackboard artifact {0} TagNameDialog.title.text=New Tag TagNameDialog.JOptionPane.tagNameIllegalCharacters.message=Tag name may not contain any of the following symbols\: \\ \: * ? " < > | , ; TagNameDialog.JOptionPane.tagNameIllegalCharacters.title=Invalid character in tag name From 4b9159b22ebccb2f1cf5bb53e31c236bf0659dee Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 12 Jul 2018 15:01:17 -0400 Subject: [PATCH 48/55] Fix jitter in file types tree caused by missing space --- Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java index 6a304d7b86..aa1d0a04ec 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java @@ -249,7 +249,7 @@ public final class FileTypes implements AutopsyVisitableItem { if (typesRoot.showCounts) { //only show "(counting...)" the first time, otherwise it is distracting. setDisplayName(getDisplayNameBase() + ((childCount < 0) ? Bundle.FileTypes_bgCounting_placeholder() - : ("(" + childCount + ")"))); //NON-NLS + : (" (" + childCount + ")"))); //NON-NLS new SwingWorker() { @Override protected Long doInBackground() throws Exception { From a50cf3da86b93788e884c10aa6feaa25f6268bf7 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 11:36:04 -0400 Subject: [PATCH 49/55] User searches appear in report (no heading yet) --- .../autopsy/report/TableReportGenerator.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 9d5d41c836..8c80955b23 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -555,6 +555,7 @@ class TableReportGenerator { "AND att.artifact_id = art.artifact_id " + //NON-NLS "GROUP BY list " + orderByClause; //NON-NLS + System.out.println("\n\n### keyword query: " + keywordListQuery + "\n"); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { ResultSet listsRs = dbQuery.getResultSet(); @@ -589,8 +590,9 @@ class TableReportGenerator { orderByClause = "ORDER BY list ASC, keyword ASC, parent_path ASC, name ASC, preview ASC"; //NON-NLS } // Query for keywords, grouped by list - String keywordsQuery - = "SELECT art.artifact_id, art.obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + // Query for keywords that are part of a list + String keywordListsQuery + = "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + //NON-NLS "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, blackboard_attributes AS att3, tsk_files AS f " + //NON-NLS @@ -608,9 +610,26 @@ class TableReportGenerator { + //NON-NLS "AND (att3.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ") " + //NON-NLS - "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " - + //NON-NLS - orderByClause; //NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; + + // Query for keywords that are not part of a list + String keywordAdHocQuery = + "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \"\" AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS + "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, tsk_files AS f " + // NON-NLS + "WHERE " + // NON-NLS + " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) " + // NON-NLS + "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) AND (att1.attribute_type_id = 37))) " + //NON-NLS + "AND (att1.artifact_id = art.artifact_id) " + //NON-NLS + "AND (att2.artifact_id = art.artifact_id) " + //NON-NLS + "AND (f.obj_id = art.obj_id) " + //NON-NLS + "AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID() + ") " + // NON-NLS + "AND (att2.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID() + ") " + // NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + + String keywordsQuery = keywordListsQuery + " UNION " + keywordAdHocQuery + orderByClause; + + System.out.println("\n\n### kw hits query:" + keywordsQuery); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordsQuery)) { ResultSet resultSet = dbQuery.getResultSet(); From 498c19c9dc6e4d2a9503d1ceaf99b083f137c133 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 13:20:45 -0400 Subject: [PATCH 50/55] Add fix for notable tags. --- .../autopsy/report/TableReportGenerator.java | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 8c80955b23..e8d8b067b8 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -44,6 +44,7 @@ import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import static org.sleuthkit.autopsy.casemodule.services.TagsManager.getNotableTagLabel; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -53,6 +54,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute.Type; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -538,6 +540,32 @@ class TableReportGenerator { logger.log(Level.SEVERE, "Exception while getting open case: ", ex); //NON-NLS return; } + + // First check for ad-hoc keywords + String tagIDList = ""; + if(tagNamesFilter.isEmpty()) { + System.out.println("\n## no tags selected"); + } else { + System.out.println("\n## tags: " + makeCommaSeparatedList(tagNamesFilter)); + try { + Map tagNamesMap = Case.getCurrentCaseThrows().getServices().getTagsManager().getDisplayNamesToTagNamesMap(); + for(String tagDisplayName : tagNamesFilter) { + if(tagNamesMap.containsKey(tagDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(tagDisplayName).getId(); + } else { + System.out.println("### No key for " + tagDisplayName); + } + } + } catch (NoCurrentCaseException | TskCoreException ex) { + logger.log(Level.SEVERE, "Exception while getting tag info - proceeding without tag filter: ", ex); //NON-NLS + tagIDList = ""; + } + } + System.out.println("### tag ID list: " + tagIDList); + if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { @@ -546,15 +574,18 @@ class TableReportGenerator { String keywordListQuery = "SELECT att.value_text AS list " + //NON-NLS - "FROM blackboard_attributes AS att, blackboard_artifacts AS art " + "FROM blackboard_attributes AS att, blackboard_artifacts AS art, blackboard_artifact_tags as tag " + //NON-NLS "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + //NON-NLS "AND art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + " " + //NON-NLS - "AND att.artifact_id = art.artifact_id " - + //NON-NLS - "GROUP BY list " + orderByClause; //NON-NLS + "AND att.artifact_id = art.artifact_id "; + if (! tagIDList.isEmpty()) { + keywordListQuery += "AND (art.artifact_id = tag.artifact_id) " + //NON-NLS + "AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + keywordListQuery += "GROUP BY list " + orderByClause; //NON-NLS System.out.println("\n\n### keyword query: " + keywordListQuery + "\n"); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { @@ -1642,14 +1673,15 @@ class TableReportGenerator { private HashSet getUniqueTagNames(long artifactId) throws TskCoreException { HashSet uniqueTagNames = new HashSet<>(); - String query = "SELECT display_name, artifact_id FROM tag_names AS tn, blackboard_artifact_tags AS bat " + String query = "SELECT display_name, artifact_id, knownStatus FROM tag_names AS tn, blackboard_artifact_tags AS bat " + //NON-NLS "WHERE tn.tag_name_id = bat.tag_name_id AND bat.artifact_id = " + artifactId; //NON-NLS try (SleuthkitCase.CaseDbQuery dbQuery = Case.getCurrentCaseThrows().getSleuthkitCase().executeQuery(query)) { ResultSet tagNameRows = dbQuery.getResultSet(); while (tagNameRows.next()) { - uniqueTagNames.add(tagNameRows.getString("display_name")); //NON-NLS + String notableString = tagNameRows.getInt("knownStatus") == TskData.FileKnown.BAD.ordinal() ? getNotableTagLabel() : ""; + uniqueTagNames.add(tagNameRows.getString("display_name") + notableString); //NON-NLS } } catch (TskCoreException | SQLException | NoCurrentCaseException ex) { throw new TskCoreException("Error getting tag names for artifact: ", ex); From 478cf84e5d11d4ba74b71c56c30d171f06214dfe Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 13 Jul 2018 14:02:25 -0400 Subject: [PATCH 51/55] Fixed core window opening condition. --- .../sleuthkit/autopsy/casemodule/Case.java | 15 ++-- .../DirectoryTreeTopComponent.java | 87 ++++++++++--------- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index a9bbb39907..9019ae162d 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -292,9 +292,10 @@ public class Case { */ ADDING_DATA_SOURCE_FAILED, /** - * A new data source has been added to the current case. The old value - * of the PropertyChangeEvent is null, the new value is the newly-added - * data source (type: Content). Cast the PropertyChangeEvent to + * A new data source or series of data sources have been added to the + * current case. The old value of the PropertyChangeEvent is null, the + * new value is the newly-added data source (type: Content). Cast the + * PropertyChangeEvent to * org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent to * access additional event data. */ @@ -1099,6 +1100,10 @@ public class Case { /* * Open the top components (windows within the main application * window). + * + * Note: If the core windows are not opened here, they will be + * opened via the DirectoryTreeTopComponent 'propertyChange()' + * method on a DATA_SOURCE_ADDED event. */ if (newCurrentCase.hasData()) { CoreComponentControl.openCoreWindows(); @@ -1351,11 +1356,11 @@ public class Case { public String getReportDirectory() { return getOrCreateSubdirectory(REPORTS_FOLDER); } - + /** * Gets the full path to the config directory for this case, creating it if * it does not exist. - * + * * @return The config directory path. */ public String getConfigDirectory() { diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 97b99103bd..bcbb72acb5 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -135,7 +135,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat this.forwardList = new LinkedList<>(); backButton.setEnabled(false); forwardButton.setEnabled(false); - + groupByDatasourceCheckBox.setSelected(UserPreferences.groupItemsInTreeByDatasource()); } @@ -159,7 +159,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } }); - + Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.DATA_SOURCE_ADDED), this); this.em.addPropertyChangeListener(this); IngestManager.getInstance().addIngestJobEventListener(this); @@ -379,10 +379,11 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat public int getPersistenceType() { return TopComponent.PERSISTENCE_NEVER; } - + /** - * Ask the user if they want to group by data source when opening a large case. - * + * Ask the user if they want to group by data source when opening a large + * case. + * * @param currentCase * @param dataSourceCount */ @@ -410,7 +411,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // Save the response Properties props = new Properties(); - if(dialog.groupByDataSourceSelected()) { + if (dialog.groupByDataSourceSelected()) { props.setProperty("groupByDataSource", "true"); } else { props.setProperty("groupByDataSource", "false"); @@ -423,7 +424,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } } - + /** * Called only when top component was closed on all workspaces before and * now is opened for the first time on some workspace. The intent is to @@ -432,8 +433,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * here. */ @NbBundle.Messages({"# {0} - dataSourceCount", - "DirectoryTreeTopComponent.componentOpened.groupDataSources.text=This case contains {0} data sources. Would you like to group by data source for faster loading?", - "DirectoryTreeTopComponent.componentOpened.groupDataSources.title=Group by data source?"}) + "DirectoryTreeTopComponent.componentOpened.groupDataSources.text=This case contains {0} data sources. Would you like to group by data source for faster loading?", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.title=Group by data source?"}) @Override public void componentOpened() { // change the cursor to "waiting cursor" for this operation @@ -462,21 +463,21 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } else { ModuleSettings.setConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME, String.valueOf(threshold)); } - + try { int dataSourceCount = currentCase.getDataSources().size(); - if (! UserPreferences.groupItemsInTreeByDatasource() && - dataSourceCount > threshold) { + if (!UserPreferences.groupItemsInTreeByDatasource() + && dataSourceCount > threshold) { promptForDataSourceGrouping(currentCase, dataSourceCount); } } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error loading data sources", ex); } } - + // if there's at least one image, load the image and open the top componen autopsyTreeChildrenFactory = new AutopsyTreeChildrenFactory(); - autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); + autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); Node root = new AbstractNode(autopsyTreeChildren) { //JIRA-2807: What is the point of these overrides? /** @@ -535,9 +536,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat tree.collapseNode(views); } /* - * JIRA-2806: What is this supposed to do? Right now it selects - * the data sources node, but the comment seems to indicate - * it is supposed to select the first datasource. + * JIRA-2806: What is this supposed to do? Right now it + * selects the data sources node, but the comment seems to + * indicate it is supposed to select the first datasource. */ // select the first image node, if there is one // (this has to happen after dataResult is opened, because the event @@ -566,7 +567,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // dataResult active) try { Node[] selections = get(); - if (selections != null && selections.length > 0){ + if (selections != null && selections.length > 0) { em.setSelectedNodes(selections); } } catch (PropertyVetoException ex) { @@ -674,15 +675,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } /** - * The "listener" that listens to any changes made in the Case.java class. - * It will do something based on the changes in the Case.java class. + * The "listener" that monitors changes made in the Case class. This serves + * the purpose of keeping the UI in sync with the data as it changes. * - * @param evt the property change event + * @param event The property change event. */ @Override - public void propertyChange(PropertyChangeEvent evt) { + public void propertyChange(PropertyChangeEvent event) { if (RuntimeProperties.runningWithGUI()) { - String changed = evt.getPropertyName(); + String changed = event.getPropertyName(); if (changed.equals(Case.Events.CURRENT_CASE.toString())) { // changed current case // When a case is closed, the old value of this property is the // closed Case object and the new value is null. When a case is @@ -692,15 +693,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // opened events instead of property change events would be a better // solution. Either way, more probably needs to be done to clean up // data model objects when a case is closed. - if (evt.getOldValue() != null && evt.getNewValue() == null) { + if (event.getOldValue() != null && event.getNewValue() == null) { // The current case has been closed. Reset the ExplorerManager. SwingUtilities.invokeLater(() -> { Node emptyNode = new AbstractNode(Children.LEAF); em.setRootContext(emptyNode); }); - } else if (evt.getNewValue() != null) { + } else if (event.getNewValue() != null) { // A new case has been opened. Reset the ExplorerManager. - Case newCase = (Case) evt.getNewValue(); + Case newCase = (Case) event.getNewValue(); final String newCaseName = newCase.getName(); SwingUtilities.invokeLater(() -> { em.getRootContext().setName(newCaseName); @@ -724,20 +725,27 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * already closed. */ try { - Case currentCase = Case.getCurrentCaseThrows(); - // We only need to trigger openCoreWindows() when the - // first data source is added. - if (currentCase.getDataSources().size() == 1) { + Case.getCurrentCaseThrows(); + /* + * In case the Case 'updateGUIForCaseOpened()' method hasn't + * already done so, open the tree and all other core + * windows. + * + * TODO: (JIRA-4053) DirectoryTreeTopComponent should not be + * responsible for opening core windows. Consider moving + * this to the Case class. + */ + if (!this.isOpened()) { SwingUtilities.invokeLater(CoreComponentControl::openCoreWindows); } - } catch (NoCurrentCaseException | TskCoreException notUsed) { + } catch (NoCurrentCaseException notUsed) { /** * Case is closed, do nothing. */ } } // change in node selection else if (changed.equals(ExplorerManager.PROP_SELECTED_NODES)) { - respondSelection((Node[]) evt.getOldValue(), (Node[]) evt.getNewValue()); + respondSelection((Node[]) event.getOldValue(), (Node[]) event.getNewValue()); } else if (changed.equals(IngestManager.IngestModuleEvent.DATA_ADDED.toString())) { // nothing to do here. // all nodes should be listening for these events and update accordingly. @@ -882,7 +890,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat /** * Rebuilds the autopsy tree. - * + * * Does nothing if there is no open case. */ private void rebuildTree() { @@ -895,9 +903,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat return; } if (null == currentCase || currentCase.hasData() == false) { - return; + return; } - + // refresh all children of the root. autopsyTreeChildrenFactory.refreshChildren(); @@ -924,14 +932,14 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } }.execute(); } - + /** * Selects the first node in the tree. - * + * */ - private void selectFirstChildNode () { + private void selectFirstChildNode() { Children rootChildren = em.getRootContext().getChildren(); - + if (rootChildren.getNodesCount() > 0) { Node firstNode = rootChildren.getNodeAt(0); if (firstNode != null) { @@ -940,6 +948,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } } + /** * Set the selected node using a path to a previously selected node. * From 9ef33ab771bf8343d062b934fc6ee6938bf67d21 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 14:16:35 -0400 Subject: [PATCH 52/55] Added user searches to table of contents when applicable Added fix for Notable tags Cleanup --- .../autopsy/report/TableReportGenerator.java | 61 +++++++++++++++---- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index e8d8b067b8..59951df1f9 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -541,12 +541,9 @@ class TableReportGenerator { return; } - // First check for ad-hoc keywords + // Get a list of all selected tag IDs String tagIDList = ""; - if(tagNamesFilter.isEmpty()) { - System.out.println("\n## no tags selected"); - } else { - System.out.println("\n## tags: " + makeCommaSeparatedList(tagNamesFilter)); + if( ! tagNamesFilter.isEmpty()) { try { Map tagNamesMap = Case.getCurrentCaseThrows().getServices().getTagsManager().getDisplayNamesToTagNamesMap(); for(String tagDisplayName : tagNamesFilter) { @@ -556,7 +553,16 @@ class TableReportGenerator { } tagIDList += tagNamesMap.get(tagDisplayName).getId(); } else { - System.out.println("### No key for " + tagDisplayName); + // If the tag name ends with "(Notable)", try stripping that off + if(tagDisplayName.endsWith(getNotableTagLabel())) { + String editedDisplayName = tagDisplayName.substring(0, tagDisplayName.length() - getNotableTagLabel().length()); + if(tagNamesMap.containsKey(editedDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(editedDisplayName).getId(); + } + } } } } catch (NoCurrentCaseException | TskCoreException ex) { @@ -564,8 +570,35 @@ class TableReportGenerator { tagIDList = ""; } } - System.out.println("### tag ID list: " + tagIDList); + // Check if there are any ad-hoc results + String adHocCountQuery = "SELECT COUNT(*) FROM " + //NON-NLS + "(SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 ";//NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + adHocCountQuery += "WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += " AND (art.artifact_id = tag.artifact_id) AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + adHocCountQuery += "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ")) "; //NON-NLS + + int adHocCount = 0; + try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(adHocCountQuery)) { + ResultSet adHocCountResultSet = dbQuery.getResultSet(); + if (adHocCountResultSet.next()) { + adHocCount = adHocCountResultSet.getInt(1); //NON-NLS + } else { + throw new TskCoreException("Error counting ad hoc keywords"); + } + } catch (TskCoreException | SQLException ex) { + errorList.add(NbBundle.getMessage(this.getClass(), "ReportGenerator.errList.failedQueryKWLists")); + logger.log(Level.SEVERE, "Failed to count ad hoc searches with query " + adHocCountQuery, ex); //NON-NLS + return; + } + + // Create the query to get the keyword list names if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { @@ -585,9 +618,12 @@ class TableReportGenerator { keywordListQuery += "AND (art.artifact_id = tag.artifact_id) " + //NON-NLS "AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS } + if (adHocCount > 0) { + keywordListQuery += " UNION SELECT \"\" AS list "; + } keywordListQuery += "GROUP BY list " + orderByClause; //NON-NLS - System.out.println("\n\n### keyword query: " + keywordListQuery + "\n"); + // Make the table of contents links for each list type try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { ResultSet listsRs = dbQuery.getResultSet(); List lists = new ArrayList<>(); @@ -611,6 +647,7 @@ class TableReportGenerator { return; } + // Query for keywords, grouped by list if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att3.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS + "convert_to(att1.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS @@ -620,7 +657,7 @@ class TableReportGenerator { } else { orderByClause = "ORDER BY list ASC, keyword ASC, parent_path ASC, name ASC, preview ASC"; //NON-NLS } - // Query for keywords, grouped by list + // Query for keywords that are part of a list String keywordListsQuery = "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " @@ -648,9 +685,9 @@ class TableReportGenerator { "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \"\" AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, tsk_files AS f " + // NON-NLS "WHERE " + // NON-NLS - " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) " + // NON-NLS + " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " + // NON-NLS "EXCEPT " + // NON-NLS - "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = 9) AND (att1.attribute_type_id = 37))) " + //NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + "))) " + //NON-NLS "AND (att1.artifact_id = art.artifact_id) " + //NON-NLS "AND (att2.artifact_id = art.artifact_id) " + //NON-NLS "AND (f.obj_id = art.obj_id) " + //NON-NLS @@ -659,8 +696,6 @@ class TableReportGenerator { "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS String keywordsQuery = keywordListsQuery + " UNION " + keywordAdHocQuery + orderByClause; - - System.out.println("\n\n### kw hits query:" + keywordsQuery); try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordsQuery)) { ResultSet resultSet = dbQuery.getResultSet(); From 414a65402aed9dc6e269ef469ca68fa7a9f8106b Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Jul 2018 14:30:21 -0400 Subject: [PATCH 53/55] Cleanup --- .../sleuthkit/autopsy/report/TableReportGenerator.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 59951df1f9..f87851fe87 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -607,9 +607,11 @@ class TableReportGenerator { String keywordListQuery = "SELECT att.value_text AS list " + //NON-NLS - "FROM blackboard_attributes AS att, blackboard_artifacts AS art, blackboard_artifact_tags as tag " - + //NON-NLS - "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + "FROM blackboard_attributes AS att, blackboard_artifacts AS art "; // NON-NLS + if(! tagIDList.isEmpty()) { + keywordListQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + keywordListQuery += "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + //NON-NLS "AND art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + " " + //NON-NLS From 710f129d00f9e0b15ffdba7ec40d49391d319554 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 13 Jul 2018 17:00:41 -0400 Subject: [PATCH 54/55] Update DirectoryTreeTopComponent.java --- .../autopsy/directorytree/DirectoryTreeTopComponent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index bcbb72acb5..95ba2f110c 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -733,7 +733,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * * TODO: (JIRA-4053) DirectoryTreeTopComponent should not be * responsible for opening core windows. Consider moving - * this to the Case class. + * this elsewhere. */ if (!this.isOpened()) { SwingUtilities.invokeLater(CoreComponentControl::openCoreWindows); From e3eb0918fbae9b21e28c68fb3182a34efe991b34 Mon Sep 17 00:00:00 2001 From: esaunders Date: Mon, 16 Jul 2018 11:02:59 -0400 Subject: [PATCH 55/55] Query node status every 5 minutes and only display nodes that have been heard from in the last 15 minutes. --- .../autoingest/AutoIngestMonitor.java | 60 ++++++++++++------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index 1b1b1bd613..876a56c47e 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.nio.file.Path; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -116,11 +117,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } catch (AutopsyEventException ex) { throw new AutoIngestMonitorException("Failed to open auto ingest event channel", ex); //NON-NLS } - coordSvcQueryExecutor.scheduleWithFixedDelay(new CoordinationServiceQueryTask(), 0, CORRD_SVC_QUERY_INERVAL_MINS, TimeUnit.MINUTES); + coordSvcQueryExecutor.scheduleWithFixedDelay(new StateRefreshTask(), 0, CORRD_SVC_QUERY_INERVAL_MINS, TimeUnit.MINUTES); eventPublisher.addSubscriber(EVENT_LIST, this); - // Publish an event that asks running nodes to send their state. - eventPublisher.publishRemotely(new AutoIngestRequestNodeStateEvent(AutoIngestManager.Event.REPORT_STATE)); + refreshNodeState(); } /** @@ -226,7 +226,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @param event A job/case prioritization event. */ private void handleCasePrioritizationEvent(AutoIngestCasePrioritizedEvent event) { - coordSvcQueryExecutor.submit(new CoordinationServiceQueryTask()); + coordSvcQueryExecutor.submit(new StateRefreshTask()); } /** @@ -235,7 +235,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @param event A job/case deletion event. */ private void handleCaseDeletedEvent(AutoIngestCaseDeletedEvent event) { - coordSvcQueryExecutor.submit(new CoordinationServiceQueryTask()); + coordSvcQueryExecutor.submit(new StateRefreshTask()); } /** @@ -297,7 +297,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @return */ List getNodeStates() { - return nodeStates.values().stream().collect(Collectors.toList()); + // We only report the state for nodes for which we have received + // a 'state' event in the last 15 minutes. + return nodeStates.values() + .stream() + .filter(s -> s.getLastSeenTime().isAfter(Instant.now().minus(Duration.ofMinutes(15)))) + .collect(Collectors.toList()); } /** @@ -313,6 +318,14 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } } + /** + * Ask running auto ingest nodes to report their state. + */ + private void refreshNodeState() { + // Publish an event that asks running nodes to send their state. + eventPublisher.publishRemotely(new AutoIngestRequestNodeStateEvent(AutoIngestManager.Event.REPORT_STATE)); + } + /** * Gets a new snapshot of the pending jobs queue, running jobs list, and * completed jobs list for an auto ingest cluster. @@ -735,25 +748,26 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } /** - * A task that queries the coordination service for auto ingest manifest - * node data and converts it to auto ingest jobs for publication top its - * observers. + * A task that updates the state maintained by the monitor. + * At present this includes auto ingest job and auto ingest node data. + * The job data is refreshed by querying the coordination service for + * auto ingest manifest nodes. + * The auto ingest node data is refreshed by publishing a message asking + * all nodes to report their state. */ - private final class CoordinationServiceQueryTask implements Runnable { + private final class StateRefreshTask implements Runnable { - /** - * Queries the coordination service for auto ingest manifest node data - * and converts it to auto ingest jobs for publication top its - * observers. - */ @Override public void run() { if (!Thread.currentThread().isInterrupted()) { - synchronized (jobsLock) { - jobsSnapshot = queryCoordinationService(); - setChanged(); - notifyObservers(); - } + // Query coordination service for jobs data. + refreshJobsSnapshot(); + + // Ask running auto ingest nodes to report their status. + refreshNodeState(); + + setChanged(); + notifyObservers(); } } @@ -867,6 +881,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen private final String nodeName; private final State nodeState; + private final Instant lastSeenTime; AutoIngestNodeState(String name, Event event) { nodeName = name; @@ -896,6 +911,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen nodeState = State.UNKNOWN; break; } + lastSeenTime = Instant.now(); } String getName() { @@ -905,6 +921,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen State getState() { return nodeState; } + + Instant getLastSeenTime() { + return lastSeenTime; + } } /**