diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java index e515225478..d2283dc9c1 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSet.java @@ -53,8 +53,11 @@ public final class FilesSet implements Serializable { private final Map rules; - private final Map inclusiveRules; - private final Map exclusiveRules; + // NOTE: these fields are derived from 'rules' which may happen at runtime + // due to deserialization of older FilesSet objects before inclusive and + // exclusive rules existed. + private Map inclusiveRules; + private Map exclusiveRules; /** * Constructs an interesting files set. @@ -108,16 +111,24 @@ public final class FilesSet implements Serializable { this.ignoreUnallocatedSpace = ignoreUnallocatedSpace; this.rules = rules == null ? Collections.emptyMap() : new HashMap<>(rules); + divideInclusiveExclusive(); + } + + /** + * Divides rules into inclusive and exclusive rules setting object fields. + */ + private void divideInclusiveExclusive() { Map inclusiveRules = new HashMap<>(); Map exclusiveRules = new HashMap<>(); - for (Entry ruleEntry : rules.entrySet()) { - if (ruleEntry.getValue().isExclusive()) { - exclusiveRules.put(ruleEntry.getKey(), ruleEntry.getValue()); - } else { - inclusiveRules.put(ruleEntry.getKey(), ruleEntry.getValue()); + if (this.rules != null) { + for (Entry ruleEntry : this.rules.entrySet()) { + if (ruleEntry.getValue().isExclusive()) { + exclusiveRules.put(ruleEntry.getKey(), ruleEntry.getValue()); + } else { + inclusiveRules.put(ruleEntry.getKey(), ruleEntry.getValue()); + } } } - this.inclusiveRules = inclusiveRules; this.exclusiveRules = exclusiveRules; } @@ -211,6 +222,10 @@ public final class FilesSet implements Serializable { || file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS))) { return null; } + + if (inclusiveRules == null || exclusiveRules == null) { + divideInclusiveExclusive(); + } String ruleName; diff --git a/Core/src/org/sleuthkit/autopsy/report/infrastructure/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/infrastructure/TableReportGenerator.java index c2edc9cb46..8a591d693e 100644 --- a/Core/src/org/sleuthkit/autopsy/report/infrastructure/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/infrastructure/TableReportGenerator.java @@ -38,6 +38,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.logging.Level; +import java.util.stream.Collectors; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -56,6 +57,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.Type; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; +import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; @@ -1222,10 +1224,14 @@ class TableReportGenerator { * * @return a list of the filtered tags. */ - private List getFilteredArtifacts(BlackboardArtifact.Type type, HashSet tagNamesFilter) { + private List getFilteredArtifacts(BlackboardArtifact.Type type, HashSet tagNamesFilter) { List artifacts = new ArrayList<>(); try { - for (BlackboardArtifact artifact : Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard().getArtifacts(Collections.singletonList(type), settings.getSelectedDataSources())) { + List dsIds = settings.getSelectedDataSources() != null + ? settings.getSelectedDataSources() + : Case.getCurrentCaseThrows().getSleuthkitCase().getDataSources().stream().map(DataSource::getId).collect(Collectors.toList()); + + for (BlackboardArtifact artifact : Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard().getArtifacts(Collections.singletonList(type), dsIds)) { if (shouldFilterFromReport(artifact)) { continue; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java index c9aa061400..ce6c23f57d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java @@ -112,9 +112,9 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { } if (filesIndexed == 0) { if (isIngestRunning) { + // ELTODO this message should be dependent on whether Solr indexing is enabled or not KeywordSearchUtil.displayDialog(keywordSearchErrorDialogHeader, NbBundle.getMessage(this.getClass(), - "AbstractKeywordSearchPerformer.search.noFilesInIdxMsg", - KeywordSearchSettings.getUpdateFrequency().getTime()), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); + "AbstractKeywordSearchPerformer.search.noFilesInIdxMsg"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); } else { KeywordSearchUtil.displayDialog(keywordSearchErrorDialogHeader, NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.noFilesIdxdMsg"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 831379e537..4c1a3ee79a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -39,8 +39,8 @@ AbstractKeywordSearchPerformer.search.invalidSyntaxHeader=Invalid query statemen AbstractKeywordSearchPerformer.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress AbstractKeywordSearchPerformer.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway? AbstractKeywordSearchPerformer.search.emptyKeywordErrorBody=Keyword list is empty, please add at least one keyword to the list -AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes. -AbstractKeywordSearchPerformer.search.noFilesIdxdMsg=No files were indexed.
Re-ingest the image with the Keyword Search Module enabled. +AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=No files are in index yet.
If Solr keyword search indexing was enabled, wait for ingest to complete +AbstractKeywordSearchPerformer.search.noFilesIdxdMsg=No files were indexed.
Re-ingest the image with the Keyword Search Module and Solr indexing enabled. ExtractedContentViewer.toolTip=Displays extracted text from files and keyword-search results. Requires Keyword Search ingest to be run on a file to activate this viewer. ExtractedContentViewer.getTitle=Indexed Text HighlightedMatchesSource.toString=Search Results @@ -211,11 +211,6 @@ KeywordSearchGlobalLanguageSettingsPanel.enableUTF8Checkbox.text=Enable UTF8 tex KeywordSearchGlobalLanguageSettingsPanel.ingestSettingsLabel.text=Ingest settings for string extraction from unknown file types (changes effective on next ingest): KeywordSearchGlobalLanguageSettingsPanel.enableUTF16Checkbox.text=Enable UTF16LE and UTF16BE string extraction KeywordSearchGlobalLanguageSettingsPanel.languagesLabel.text=Enabled scripts (languages): -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.toolTipText=20 mins. (fastest ingest time) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.text=20 minutes (slowest feedback, fastest ingest) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.toolTipText=10 minutes (faster overall ingest time than default) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.text=10 minutes (slower feedback, faster ingest) -KeywordSearchGlobalSearchSettingsPanel.frequencyLabel.text=Results update frequency during ingest: KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.toolTipText=Requires Hash Set service to had run previously, or be selected for next ingest. KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text=Do not add files in NSRL (known files) to keyword index during ingest KeywordSearchGlobalSearchSettingsPanel.informationLabel.text=Information @@ -224,11 +219,7 @@ KeywordSearchGlobalSearchSettingsPanel.filesIndexedValue.text=0 KeywordSearchGlobalSearchSettingsPanel.filesIndexedLabel.text=Files in keyword index: KeywordSearchGlobalSearchSettingsPanel.showSnippetsCB.text=Show Keyword Preview in Keyword Search Results (will result in longer search times) KeywordSearchGlobalSearchSettingsPanel.chunksValLabel.text=0 -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.toolTipText=1 minute (overall ingest time will be longest) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1 minute (faster feedback, longest ingest) KeywordSearchGlobalSearchSettingsPanel.chunksLabel.text=Chunks in keyword index: -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5 minutes (overall ingest time will be longer) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5 minutes (default) KeywordSearchIngestModule.regExpHitLbl=Reg Ex hit: KeywordSearchIngestModule.kwHitLbl=Keyword hit: KeywordSearchIngestModule.kwHitThLbl=Keyword @@ -254,8 +245,6 @@ KeywordSearchListsManagementPanel.newKeywordListDescription2=Keyword list <{0}> KeywordSearchModuleFactory.getIngestJobSettingsPanel.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings KeywordSearchModuleFactory.createFileIngestModule.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings SearchRunner.Searcher.done.err.msg=Error performing keyword search -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText=Fastest overall, but no results until the end -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text=No periodic searches SolrConnectionCheck.HostnameOrPort=Invalid hostname and/or port number. SolrConnectionCheck.Hostname=Invalid hostname. SolrConnectionCheck.MissingHostname=Missing hostname. diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 1509902680..f6b75b139a 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -18,9 +18,8 @@ ExtractAllTermsReport.getName.text=Extract Unique Words # {0} - Number of extracted terms ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms... ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and unique word extraction might yield incomplete results.
Do you want to proceed with unique word extraction anyway? -# {0} - Keyword search commit frequency -ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. Try again later. Index is updated every {0} minutes. -ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try again later +ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. If Solr keyword search indexing and Solr indexing were enabled, wait for ingest to complete. +ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Re-ingest the image with the Keyword Search Module and Solr indexing enabled. ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress ExtractAllTermsReport.startExport=Starting Unique Word Extraction ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait @@ -91,7 +90,7 @@ AbstractKeywordSearchPerformer.search.invalidSyntaxHeader=Invalid query statemen AbstractKeywordSearchPerformer.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress AbstractKeywordSearchPerformer.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway? AbstractKeywordSearchPerformer.search.emptyKeywordErrorBody=Keyword list is empty, please add at least one keyword to the list -AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes. +AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=No files are in index yet.
If Solr keyword search indexing was enabled, wait for ingest to complete AbstractKeywordSearchPerformer.search.noFilesIdxdMsg=No files were indexed.
Re-ingest the image with the Keyword Search Module enabled. ExtractedContentViewer.toolTip=Displays extracted text from files and keyword-search results. Requires Keyword Search ingest to be run on a file to activate this viewer. ExtractedContentViewer.getTitle=Indexed Text @@ -227,7 +226,6 @@ KeywordSearchSettings.properties_options.text={0}_Options KeywordSearchSettings.propertiesNSRL.text={0}_NSRL KeywordSearchSettings.propertiesScripts.text={0}_Scripts NoOpenCoreException.err.noOpenSorlCore.msg=No currently open Solr core. -SearchRunner.query.exception.msg=Error performing query: # {0} - colelction name Server.deleteCore.exception.msg=Failed to delete Solr colelction {0} Server.exceptionMessage.unableToBackupCollection=Unable to backup Solr collection @@ -272,11 +270,6 @@ KeywordSearchGlobalLanguageSettingsPanel.enableUTF8Checkbox.text=Enable UTF8 tex KeywordSearchGlobalLanguageSettingsPanel.ingestSettingsLabel.text=Ingest settings for string extraction from unknown file types (changes effective on next ingest): KeywordSearchGlobalLanguageSettingsPanel.enableUTF16Checkbox.text=Enable UTF16LE and UTF16BE string extraction KeywordSearchGlobalLanguageSettingsPanel.languagesLabel.text=Enabled scripts (languages): -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.toolTipText=20 mins. (fastest ingest time) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.text=20 minutes (slowest feedback, fastest ingest) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.toolTipText=10 minutes (faster overall ingest time than default) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.text=10 minutes (slower feedback, faster ingest) -KeywordSearchGlobalSearchSettingsPanel.frequencyLabel.text=Results update frequency during ingest: KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.toolTipText=Requires Hash Set service to had run previously, or be selected for next ingest. KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text=Do not add files in NSRL (known files) to keyword index during ingest KeywordSearchGlobalSearchSettingsPanel.informationLabel.text=Information @@ -285,11 +278,7 @@ KeywordSearchGlobalSearchSettingsPanel.filesIndexedValue.text=0 KeywordSearchGlobalSearchSettingsPanel.filesIndexedLabel.text=Files in keyword index: KeywordSearchGlobalSearchSettingsPanel.showSnippetsCB.text=Show Keyword Preview in Keyword Search Results (will result in longer search times) KeywordSearchGlobalSearchSettingsPanel.chunksValLabel.text=0 -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.toolTipText=1 minute (overall ingest time will be longest) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1 minute (faster feedback, longest ingest) KeywordSearchGlobalSearchSettingsPanel.chunksLabel.text=Chunks in keyword index: -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5 minutes (overall ingest time will be longer) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5 minutes (default) KeywordSearchIngestModule.regExpHitLbl=Reg Ex hit: KeywordSearchIngestModule.kwHitLbl=Keyword hit: KeywordSearchIngestModule.kwHitThLbl=Keyword @@ -315,8 +304,6 @@ KeywordSearchListsManagementPanel.newKeywordListDescription2=Keyword list <{0}> KeywordSearchModuleFactory.getIngestJobSettingsPanel.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings KeywordSearchModuleFactory.createFileIngestModule.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings SearchRunner.Searcher.done.err.msg=Error performing keyword search -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText=Fastest overall, but no results until the end -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text=No periodic searches Server.status.failed.msg=Local Solr server did not respond to status request. This may be because the server failed to start or is taking too long to initialize. SolrConnectionCheck.HostnameOrPort=Invalid hostname and/or port number. SolrConnectionCheck.Hostname=Invalid hostname. diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties index 46af934d7c..83e614bc93 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties @@ -7,8 +7,6 @@ AbstractKeywordSearchPerformer.search.dialogErrorHeader=\u30ad\u30fc\u30ef\u30fc AbstractKeywordSearchPerformer.search.emptyKeywordErrorBody=\u30ad\u30fc\u30ef\u30fc\u30c9\u30ea\u30b9\u30c8\u304c\u7a7a(\u672a\u5165\u529b)\u3067\u3059\u3002\u5c11\u306a\u304f\u3068\u30821\u3064\u306e\u30ad\u30fc\u30ef\u30fc\u30c9\u3092\u30ea\u30b9\u30c8\u306b\u8ffd\u52a0\u3057\u3066\u304f\u3060\u3055\u3044\u3002 AbstractKeywordSearchPerformer.search.ingestInProgressBody=\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u73fe\u5728\u5b9f\u884c\u4e2d\u3067\u3059\u3002
\u3059\u3079\u3066\u306e\u30d5\u30a1\u30a4\u30eb\u304c\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3055\u308c\u306a\u304b\u3063\u305f\u305f\u3081\u3001\u3053\u306e\u691c\u7d22\u306f\u4e0d\u5b8c\u5168\u306a\u7d50\u679c\u3092\u751f\u6210\u3059\u308b\u53ef\u80fd\u6027\u304c\u3042\u308a\u307e\u3059\u3002
\u305d\u308c\u3067\u3082\u3053\u306e\u691c\u7d22\u3092\u7d9a\u884c\u3057\u307e\u3059\u304b? AbstractKeywordSearchPerformer.search.invalidSyntaxHeader=\u7121\u52b9\u306a\u30af\u30a8\u30ea\u30fb\u30b9\u30c6\u30fc\u30c8\u30e1\u30f3\u30c8\u3002 \u5185\u5bb9\u304c\u6b63\u898f\u8868\u73fe\u306e\u5834\u5408\u3001Lucene\u6b63\u898f\u8868\u73fe\u30d1\u30bf\u30fc\u30f3\u306e\u307f\u304c\u30b5\u30dd\u30fc\u30c8\u3055\u308c\u307e\u3059\u3002 POSIX\u6587\u5b57\u30af\u30e9\u30b9\uff08\\ n\u3084\\ w\u306a\u3069\uff09\u306f\u7121\u52b9\u3067\u3059\u3002 -AbstractKeywordSearchPerformer.search.noFilesIdxdMsg=\u30d5\u30a1\u30a4\u30eb\u304c\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002
\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u6709\u52b9\u306b\u3057\u305f\u72b6\u614b\u3067\u30a4\u30e1\u30fc\u30b8\u3092\u518d\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3057\u3066\u304f\u3060\u3055\u3044\u3002 -AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=\u307e\u3060\u30d5\u30a1\u30a4\u30eb\u304c\u7d22\u5f15\u306b\u542b\u307e\u308c\u3066\u3044\u307e\u305b\u3093\u3002
\u5f8c\u3067\u3082\u3046\u4e00\u5ea6\u304a\u8a66\u3057\u304f\u3060\u3055\u3044\u3002 \u7d22\u5f15\u306f {0} \u5206\u3054\u3068\u306b\u66f4\u65b0\u3055\u308c\u307e\u3059\u3002 AbstractKeywordSearchPerformer.search.searchIngestInProgressTitle=\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u9032\u884c\u4e2d\u3067\u3059 AccountsText.creditCardNumber=\u30af\u30ec\u30b8\u30c3\u30c8\u30ab\u30fc\u30c9\u756a\u53f7 AccountsText.creditCardNumbers=\u30af\u30ec\u30b8\u30c3\u30c8\u30ab\u30fc\u30c9\u756a\u53f7 @@ -55,8 +53,6 @@ ExtractAllTermsReport.exportComplete=\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\ ExtractAllTermsReport.getName.text=\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\u3092\u62bd\u51fa\u3059\u308b ExtractAllTermsReport.numberExtractedTerms=\u62bd\u51fa\u3055\u308c\u305f{0}\u7528\u8a9e... ExtractAllTermsReport.search.ingestInProgressBody= \u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u8aad\u8fbc\u306f\u73fe\u5728\u5b9f\u884c\u4e2d\u3067\u3059\u3002
\u3059\u3079\u3066\u306e\u30d5\u30a1\u30a4\u30eb\u304c\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306b\u767b\u9332\u3055\u308c\u3066\u3044\u308b\u308f\u3051\u3067\u306f\u306a\u304f\u3001\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\u3092\u62bd\u51fa\u306f\u4e0d\u5b8c\u5168\u306a\u7d50\u679c\u306b\u306a\u308b\u53ef\u80fd\u6027\u304c\u3042\u308a\u307e\u3059\u3002
\u305d\u308c\u3067\u3082\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\u306e\u62bd\u51fa\u3092\u7d9a\u884c\u3057\u307e\u3059\u304b\uff1f -ExtractAllTermsReport.search.noFilesInIdxMsg=\u307e\u3060\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306b\u767b\u9332\u3055\u308c\u3066\u3044\u308b\u30d5\u30a1\u30a4\u30eb\u306f\u3042\u308a\u307e\u305b\u3093\u3002 \u3042\u3068\u3067\u3082\u3046\u4e00\u5ea6\u8a66\u3057\u3066\u307f\u3066\u304f\u3060\u3055\u3044\u3002 \u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306f{0}\u5206\u3054\u3068\u306b\u66f4\u65b0\u3055\u308c\u307e\u3059\u3002 -ExtractAllTermsReport.search.noFilesInIdxMsg2=\u307e\u3060\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306b\u767b\u9332\u3055\u308c\u3066\u3044\u308b\u30d5\u30a1\u30a4\u30eb\u306f\u3042\u308a\u307e\u305b\u3093\u3002 \u3042\u3068\u3067\u3082\u3046\u4e00\u5ea6\u8a66\u3057\u3066\u307f\u3066\u304f\u3060\u3055\u3044\u3002 ExtractAllTermsReport.search.searchIngestInProgressTitle=\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u9032\u884c\u4e2d\u3067\u3059 ExtractAllTermsReport.startExport=\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\u62bd\u51fa\u306e\u958b\u59cb ExtractedContentPanel.SetMarkup.progress.loading={0} \u306e\u30c6\u30ad\u30b9\u30c8\u3092\u8aad\u307f\u8fbc\u3093\u3067\u3044\u307e\u3059 @@ -210,23 +206,12 @@ KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsLimitedOCR=\u3 KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsOCR=OCR\u6587\u5b57\u8a8d\u8b58\u3092\u6709\u52b9\u306b\u3059\u308b\uff08Windows 64\u30d3\u30c3\u30c8\u304c\u5fc5\u8981\uff09 KeywordSearchGlobalSearchSettingsPanel.filesIndexedLabel.text=\u30ad\u30fc\u30ef\u30fc\u30c9\u7d22\u5f15\u5185\u306e\u30d5\u30a1\u30a4\u30eb\: KeywordSearchGlobalSearchSettingsPanel.filesIndexedValue.text=0 -KeywordSearchGlobalSearchSettingsPanel.frequencyLabel.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u4e2d\u306e\u7d50\u679c\u66f4\u65b0\u983b\u5ea6\: KeywordSearchGlobalSearchSettingsPanel.informationLabel.text=\u60c5\u5831 KeywordSearchGlobalSearchSettingsPanel.ingestWarningLabel.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u9032\u884c\u4e2d\u3067\u3059\u3002\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u5b8c\u4e86\u3059\u308b\u307e\u3067\u4e00\u90e8\u306e\u8a2d\u5b9a\u3092\u5229\u7528\u3067\u304d\u307e\u305b\u3093\u3002 KeywordSearchGlobalSearchSettingsPanel.settingsLabel.text=\u8a2d\u5b9a KeywordSearchGlobalSearchSettingsPanel.showSnippetsCB.text=\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u7d50\u679c\u306b\u30ad\u30fc\u30ef\u30fc\u30c9\u30d7\u30ec\u30d3\u30e5\u30fc\u3092\u8868\u793a(\u691c\u7d22\u6642\u9593\u304c\u9577\u304f\u306a\u308a\u307e\u3059) KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u4e2d\u306bNSRL(\u65e2\u77e5\u306e\u30d5\u30a1\u30a4\u30eb)\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u30ad\u30fc\u30ef\u30fc\u30c9\u306b\u8ffd\u52a0\u3057\u306a\u3044\u3067\u304f\u3060\u3055\u3044 KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.toolTipText=\u30cf\u30c3\u30b7\u30e5\u30bb\u30c3\u30c8\u30b5\u30fc\u30d3\u30b9\u306b\u4ee5\u524d\u306b\u5b9f\u884c\u6e08\u307f\u3067\u3042\u308b\u3053\u3068\u3001\u307e\u305f\u306f\u6b21\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u306b\u9078\u629e\u3055\u308c\u308b\u3053\u3068\u3092\u8981\u6c42\u3057\u307e\u3059\u3002 -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.text=20\u5206(\u6700\u9045\u30d5\u30a3\u30fc\u30c9\u30d0\u30c3\u30af\u3001\u6700\u901f\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.toolTipText=20\u5206(\u6700\u901f\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u6642\u9593) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.text=10\u5206(\u3088\u308a\u9045\u3044\u30d5\u30a3\u30fc\u30c9\u30d0\u30c3\u30af\u3001\u3088\u308a\u901f\u3044\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.toolTipText=10\u5206(\u30c7\u30d5\u30a9\u30eb\u30c8\u3088\u308a\u3082\u901f\u3044\u7dcf\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u6642\u9593) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5\u5206(\u30c7\u30d5\u30a9\u30eb\u30c8) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5\u5206(\u7dcf\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u6642\u9593\u304c\u9577\u304f\u306a\u308a\u307e\u3059) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1\u5206(\u3088\u308a\u901f\u3044\u30d5\u30a3\u30fc\u30c9\u30d0\u30c3\u30af\u3001\u6700\u9577\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.toolTipText=1\u5206(\u7dcf\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u6642\u9593\u306f\u6700\u9577\u306b\u306a\u308a\u307e\u3059) -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text=\u5b9a\u671f\u691c\u7d22\u306a\u3057 -KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText=\u5168\u4f53\u3067\u6700\u901f\u3067\u3059\u304c\u3001\u6700\u5f8c\u307e\u3067\u7d50\u679c\u306f\u8868\u793a\u3055\u308c\u307e\u305b\u3093 KeywordSearchGlobalSettingsPanel.Title=\u30ad\u30fc\u30ef\u30fc\u30c9\u4e00\u62ec\u691c\u7d22\u8a2d\u5b9a KeywordSearchIngestModule.doInBackGround.displayName=\u30ad\u30fc\u30ef\u30fc\u30c9\u5b9a\u671f\u691c\u7d22 KeywordSearchIngestModule.doInBackGround.finalizeMsg=\u78ba\u5b9a diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index 7b84080217..f53dc547c5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2018 Basis Technology Corp. + * Copyright 2011-2022 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,7 +30,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.logging.Level; import javax.swing.JCheckBox; import javax.swing.JTable; import javax.swing.ListSelectionModel; @@ -143,10 +142,7 @@ class DropdownListSearchPanel extends AdHocSearchPanel { searchAddListener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { - if (ingestRunning) { - IngestSearchRunner.getInstance().addKeywordListsToAllJobs(listsTableModel.getSelectedLists()); - logger.log(Level.INFO, "Submitted enqueued lists to ingest"); //NON-NLS - } else { + if (!ingestRunning) { searchAction(e); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java index 88178b42ea..584757aa93 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2021 Basis Technology Corp. + * Copyright 2022 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -51,9 +51,8 @@ public class ExtractAllTermsReport implements GeneralReportModule { @NbBundle.Messages({ "ExtractAllTermsReport.error.noOpenCase=No currently open case.", - "# {0} - Keyword search commit frequency", - "ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. Try again later. Index is updated every {0} minutes.", - "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try again later", + "ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. If Solr keyword search indexing and Solr indexing were enabled, wait for ingest to complete.", + "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Re-ingest the image with the Keyword Search Module and Solr indexing enabled.", "ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress", "ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and unique word extraction might yield incomplete results.
Do you want to proceed with unique word extraction anyway?", "ExtractAllTermsReport.startExport=Starting Unique Word Extraction", @@ -83,7 +82,7 @@ public class ExtractAllTermsReport implements GeneralReportModule { if (filesIndexed == 0) { if (isIngestRunning) { - progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_search_noFilesInIdxMsg(KeywordSearchSettings.getUpdateFrequency().getTime())); + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_search_noFilesInIdxMsg()); } else { progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_search_noFilesInIdxMsg2()); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java deleted file mode 100755 index 9cd33a8167..0000000000 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ /dev/null @@ -1,705 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2014 - 2021 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.keywordsearch; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; -import javax.annotation.concurrent.GuardedBy; -import javax.swing.SwingUtilities; -import javax.swing.SwingWorker; -import org.netbeans.api.progress.ProgressHandle; -import org.openide.util.Cancellable; -import org.openide.util.NbBundle; -import org.openide.util.NbBundle.Messages; -import org.sleuthkit.autopsy.core.RuntimeProperties; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.coreutils.StopWatch; -import org.sleuthkit.autopsy.coreutils.ThreadConfined; -import org.sleuthkit.autopsy.ingest.IngestJobContext; -import org.sleuthkit.autopsy.ingest.IngestMessage; -import org.sleuthkit.autopsy.ingest.IngestServices; - -/** - * Performs periodic and final keyword searches for ingest jobs. Periodic - * searches are done in background tasks. This represents a careful working - * around of the contract for IngestModule.process(). Final searches are done - * synchronously in the calling thread, as required by the contract for - * IngestModule.shutDown(). - */ -final class IngestSearchRunner { - - private static final Logger logger = Logger.getLogger(IngestSearchRunner.class.getName()); - private static IngestSearchRunner instance = null; - private final IngestServices services = IngestServices.getInstance(); - private Ingester ingester = null; - private long currentUpdateIntervalMs; - private volatile boolean periodicSearchTaskRunning; - private volatile Future periodicSearchTaskHandle; - private final ScheduledThreadPoolExecutor periodicSearchTaskExecutor; - private static final int NUM_SEARCH_SCHEDULING_THREADS = 1; - private static final String SEARCH_SCHEDULER_THREAD_NAME = "periodic-search-scheduling-%d"; - private final Map jobs = new ConcurrentHashMap<>(); // Ingest job ID to search job info - private final boolean usingNetBeansGUI = RuntimeProperties.runningWithGUI(); - - /* - * Constructs a singleton object that performs periodic and final keyword - * searches for ingest jobs. Periodic searches are done in background tasks. - * This represents a careful working around of the contract for - * IngestModule.process(). Final searches are done synchronously in the - * calling thread, as required by the contract for IngestModule.shutDown(). - */ - private IngestSearchRunner() { - currentUpdateIntervalMs = ((long) KeywordSearchSettings.getUpdateFrequency().getTime()) * 60 * 1000; - ingester = Ingester.getDefault(); - periodicSearchTaskExecutor = new ScheduledThreadPoolExecutor(NUM_SEARCH_SCHEDULING_THREADS, new ThreadFactoryBuilder().setNameFormat(SEARCH_SCHEDULER_THREAD_NAME).build()); - } - - /** - * Gets the ingest search runner singleton. - * - * @return The ingest search runner. - */ - public static synchronized IngestSearchRunner getInstance() { - if (instance == null) { - instance = new IngestSearchRunner(); - } - return instance; - } - - /** - * Starts the search job for an ingest job. - * - * @param jobContext The ingest job context. - * @param keywordListNames The names of the keyword search lists for the - * ingest job. - */ - public synchronized void startJob(IngestJobContext jobContext, List keywordListNames) { - long jobId = jobContext.getJobId(); - if (jobs.containsKey(jobId) == false) { - SearchJobInfo jobData = new SearchJobInfo(jobContext, keywordListNames); - jobs.put(jobId, jobData); - } - - /* - * Keep track of the number of keyword search file ingest modules that - * are doing analysis for the ingest job, i.e., that have called this - * method. This is needed by endJob(). - */ - jobs.get(jobId).incrementModuleReferenceCount(); - - /* - * Start a periodic search task in the - */ - if ((jobs.size() > 0) && (periodicSearchTaskRunning == false)) { - currentUpdateIntervalMs = ((long) KeywordSearchSettings.getUpdateFrequency().getTime()) * 60 * 1000; - periodicSearchTaskHandle = periodicSearchTaskExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); - periodicSearchTaskRunning = true; - } - } - - /** - * Finishes a search job for an ingest job. - * - * @param jobId The ingest job ID. - */ - public synchronized void endJob(long jobId) { - /* - * Only complete the job if this is the last keyword search file ingest - * module doing annalysis for this job. - */ - SearchJobInfo job; - job = jobs.get(jobId); - if (job == null) { - return; // RJCTODO: SEVERE - } - if (job.decrementModuleReferenceCount() != 0) { - jobs.remove(jobId); - } - - /* - * Commit the index and do the final search. The final search is done in - * the ingest thread that shutDown() on the keyword search file ingest - * module, per the contract of IngestModule.shutDwon(). - */ - logger.log(Level.INFO, "Commiting search index before final search for search job {0}", job.getJobId()); //NON-NLS - commit(); - logger.log(Level.INFO, "Starting final search for search job {0}", job.getJobId()); //NON-NLS - doFinalSearch(job); - logger.log(Level.INFO, "Final search for search job {0} completed", job.getJobId()); //NON-NLS - - if (jobs.isEmpty()) { - cancelPeriodicSearchSchedulingTask(); - } - } - - /** - * Stops the search job for an ingest job. - * - * @param jobId The ingest job ID. - */ - public synchronized void stopJob(long jobId) { - logger.log(Level.INFO, "Stopping search job {0}", jobId); //NON-NLS - commit(); - - SearchJobInfo job; - job = jobs.get(jobId); - if (job == null) { - return; - } - - /* - * Request cancellation of the current keyword search, whether it is a - * preiodic search or a final search. - */ - IngestSearchRunner.Searcher currentSearcher = job.getCurrentSearcher(); - if ((currentSearcher != null) && (!currentSearcher.isDone())) { - logger.log(Level.INFO, "Cancelling search job {0}", jobId); //NON-NLS - currentSearcher.cancel(true); - } - - jobs.remove(jobId); - - if (jobs.isEmpty()) { - cancelPeriodicSearchSchedulingTask(); - } - } - - /** - * Adds the given keyword list names to the set of keyword lists to be - * searched by ALL keyword search jobs. This supports adding one or more - * keyword search lists to ingest jobs already in progress. - * - * @param keywordListNames The n ames of the additional keyword lists. - */ - public synchronized void addKeywordListsToAllJobs(List keywordListNames) { - for (String listName : keywordListNames) { - logger.log(Level.INFO, "Adding keyword list {0} to all jobs", listName); //NON-NLS - for (SearchJobInfo j : jobs.values()) { - j.addKeywordListName(listName); - } - } - } - - /** - * Commits the Solr index for the current case and publishes an event - * indicating the current number of indexed items (this is no longer just - * files). - */ - private void commit() { - ingester.commit(); - - /* - * Publish an event advertising the number of indexed items. Note that - * this is no longer the number of indexed files, since the text of many - * items in addition to files is indexed. - */ - try { - final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); - KeywordSearch.fireNumIndexedFilesChange(null, numIndexedFiles); - } catch (NoOpenCoreException | KeywordSearchModuleException ex) { - logger.log(Level.SEVERE, "Error executing Solr query for number of indexed files", ex); //NON-NLS - } - } - - /** - * Performs the final keyword search for an ingest job. The search is done - * synchronously, as required by the contract for IngestModule.shutDown(). - * - * @param job The keyword search job info. - */ - private void doFinalSearch(SearchJobInfo job) { - if (!job.getKeywordListNames().isEmpty()) { - try { - /* - * Wait for any periodic searches being done in a SwingWorker - * pool thread to finish. - */ - job.waitForCurrentWorker(); - IngestSearchRunner.Searcher finalSearcher = new IngestSearchRunner.Searcher(job, true); - job.setCurrentSearcher(finalSearcher); - /* - * Do the final search synchronously on the current ingest - * thread, per the contract specified - */ - finalSearcher.doInBackground(); - } catch (InterruptedException | CancellationException ex) { - logger.log(Level.INFO, "Final search for search job {0} interrupted or cancelled", job.getJobId()); //NON-NLS - } catch (Exception ex) { - logger.log(Level.SEVERE, String.format("Final search for search job %d failed", job.getJobId()), ex); //NON-NLS - } - } - } - - /** - * Cancels the current periodic search scheduling task. - */ - private synchronized void cancelPeriodicSearchSchedulingTask() { - if (periodicSearchTaskHandle != null) { - logger.log(Level.INFO, "No more search jobs, stopping periodic search scheduling"); //NON-NLS - periodicSearchTaskHandle.cancel(true); - periodicSearchTaskRunning = false; - } - } - - /** - * Task that runs in ScheduledThreadPoolExecutor to periodically start and - * wait for keyword search tasks for each keyword search job in progress. - * The keyword search tasks for individual ingest jobs are implemented as - * SwingWorkers to support legacy APIs. - */ - private final class PeriodicSearchTask implements Runnable { - - @Override - public void run() { - /* - * If there are no more jobs or this task has been cancelled, exit. - */ - if (jobs.isEmpty() || periodicSearchTaskHandle.isCancelled()) { - logger.log(Level.INFO, "Periodic search scheduling task has been cancelled, exiting"); //NON-NLS - periodicSearchTaskRunning = false; - return; - } - - /* - * Commit the Solr index for the current case before doing the - * searches. - */ - commit(); - - /* - * Do a keyword search for each ingest job in progress. When the - * searches are done, recalculate the "hold off" time between - * searches to prevent back-to-back periodic searches and schedule - * the nect periodic search task. - */ - final StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - for (Iterator> iterator = jobs.entrySet().iterator(); iterator.hasNext();) { - SearchJobInfo job = iterator.next().getValue(); - - if (periodicSearchTaskHandle.isCancelled()) { - logger.log(Level.INFO, "Periodic search scheduling task has been cancelled, exiting"); //NON-NLS - periodicSearchTaskRunning = false; - return; - } - - if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { - logger.log(Level.INFO, "Starting periodic search for search job {0}", job.getJobId()); - Searcher searcher = new Searcher(job, false); - job.setCurrentSearcher(searcher); - searcher.execute(); - job.setWorkerRunning(true); - try { - searcher.get(); - } catch (InterruptedException | ExecutionException ex) { - logger.log(Level.SEVERE, String.format("Error performing keyword search for ingest job %d", job.getJobId()), ex); //NON-NLS - services.postMessage(IngestMessage.createErrorMessage( - KeywordSearchModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), "SearchRunner.Searcher.done.err.msg"), ex.getMessage())); - } catch (java.util.concurrent.CancellationException ex) { - logger.log(Level.SEVERE, String.format("Keyword search for ingest job %d cancelled", job.getJobId()), ex); //NON-NLS - } - } - } - stopWatch.stop(); - logger.log(Level.INFO, "Periodic searches for all ingest jobs cumulatively took {0} secs", stopWatch.getElapsedTimeSecs()); //NON-NLS - recalculateUpdateIntervalTime(stopWatch.getElapsedTimeSecs()); // ELDEBUG - periodicSearchTaskHandle = periodicSearchTaskExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); - } - - /** - * Sets the time interval between periodic keyword searches to avoid - * running back-to-back searches. If the most recent round of searches - * took longer that 1/4 of the current interval, doubles the interval. - * - * @param lastSerchTimeSec The time in seconds used to execute the most - * recent round of keword searches. - */ - private void recalculateUpdateIntervalTime(long lastSerchTimeSec) { - if (lastSerchTimeSec * 1000 < currentUpdateIntervalMs / 4) { - return; - } - currentUpdateIntervalMs *= 2; - logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs / 1000}); - } - } - - /** - * A data structure to keep track of the keyword lists, current results, and - * search running status for an ingest job. - */ - private class SearchJobInfo { - - private final IngestJobContext jobContext; - private final long jobId; - private final long dataSourceId; - private volatile boolean workerRunning; - @GuardedBy("this") - private final List keywordListNames; - @GuardedBy("this") - private final Map> currentResults; // Keyword to object IDs of items with hits - private IngestSearchRunner.Searcher currentSearcher; - private final AtomicLong moduleReferenceCount = new AtomicLong(0); - private final Object finalSearchLock = new Object(); - - private SearchJobInfo(IngestJobContext jobContext, List keywordListNames) { - this.jobContext = jobContext; - jobId = jobContext.getJobId(); - dataSourceId = jobContext.getDataSource().getId(); - this.keywordListNames = new ArrayList<>(keywordListNames); - currentResults = new HashMap<>(); - workerRunning = false; - currentSearcher = null; - } - - private IngestJobContext getJobContext() { - return jobContext; - } - - private long getJobId() { - return jobId; - } - - private long getDataSourceId() { - return dataSourceId; - } - - private synchronized List getKeywordListNames() { - return new ArrayList<>(keywordListNames); - } - - private synchronized void addKeywordListName(String keywordListName) { - if (!keywordListNames.contains(keywordListName)) { - keywordListNames.add(keywordListName); - } - } - - private synchronized Set currentKeywordResults(Keyword k) { - return currentResults.get(k); - } - - private synchronized void addKeywordResults(Keyword k, Set resultsIDs) { - currentResults.put(k, resultsIDs); - } - - private boolean isWorkerRunning() { - return workerRunning; - } - - private void setWorkerRunning(boolean flag) { - workerRunning = flag; - } - - private synchronized IngestSearchRunner.Searcher getCurrentSearcher() { - return currentSearcher; - } - - private synchronized void setCurrentSearcher(IngestSearchRunner.Searcher searchRunner) { - currentSearcher = searchRunner; - } - - private void incrementModuleReferenceCount() { - moduleReferenceCount.incrementAndGet(); - } - - private long decrementModuleReferenceCount() { - return moduleReferenceCount.decrementAndGet(); - } - - /** - * Waits for the current search task to complete. - * - * @throws InterruptedException - */ - private void waitForCurrentWorker() throws InterruptedException { - synchronized (finalSearchLock) { - while (workerRunning) { - logger.log(Level.INFO, String.format("Waiting for previous search task for job %d to finish", jobId)); //NON-NLS - finalSearchLock.wait(); - logger.log(Level.INFO, String.format("Notified previous search task for job %d to finish", jobId)); //NON-NLS - } - } - } - - /** - * Signals any threads waiting on the current search task to complete. - */ - private void searchNotify() { - synchronized (finalSearchLock) { - workerRunning = false; - finalSearchLock.notify(); - } - } - } - - /* - * A SwingWorker responsible for searching the Solr index of the current - * case for the keywords for an ingest job. Keyword hit analysis results are - * created and posted to the blackboard and notifications are sent to the - * ingest inbox. - */ - private final class Searcher extends SwingWorker { - - /* - * Searcher has private copies/snapshots of the lists and keywords - */ - private final SearchJobInfo job; - private final List keywords; //keywords to search - private final List keywordListNames; // lists currently being searched - private final List keywordLists; - private final Map keywordToList; //keyword to list name mapping - @ThreadConfined(type = ThreadConfined.ThreadType.AWT) - private ProgressHandle progressIndicator; - private boolean finalRun = false; - - Searcher(SearchJobInfo job, boolean finalRun) { - this.job = job; - this.finalRun = finalRun; - keywordListNames = job.getKeywordListNames(); - keywords = new ArrayList<>(); - keywordToList = new HashMap<>(); - keywordLists = new ArrayList<>(); - } - - @Override - @Messages("SearchRunner.query.exception.msg=Error performing query:") - protected Object doInBackground() throws Exception { - try { - if (usingNetBeansGUI) { - /* - * If running in the NetBeans thick client application - * version of Autopsy, NetBeans progress handles (i.e., - * progress bars) are used to display search progress in the - * lower right hand corner of the main application window. - * - * A layer of abstraction to allow alternate representations - * of progress could be used here, as it is in other places - * in the application (see implementations and usage of - * org.sleuthkit.autopsy.progress.ProgressIndicator - * interface), to better decouple keyword search from the - * application's presentation layer. - */ - SwingUtilities.invokeAndWait(() -> { - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") - + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); - progressIndicator = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - if (progressIndicator != null) { - progressIndicator.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); - } - logger.log(Level.INFO, "Search cancelled by user"); //NON-NLS - new Thread(() -> { - IngestSearchRunner.Searcher.this.cancel(true); - }).start(); - return true; - } - }); - progressIndicator.start(); - progressIndicator.switchToIndeterminate(); - }); - } - - updateKeywords(); - for (Keyword keyword : keywords) { - if (isCancelled() || job.getJobContext().fileIngestIsCancelled()) { - logger.log(Level.INFO, "Cancellation requested, exiting before new keyword processed: {0}", keyword.getSearchTerm()); //NON-NLS - return null; - } - - KeywordList keywordList = keywordToList.get(keyword); - if (usingNetBeansGUI) { - String searchTermStr = keyword.getSearchTerm(); - if (searchTermStr.length() > 50) { - searchTermStr = searchTermStr.substring(0, 49) + "..."; - } - final String progressMessage = keywordList.getName() + ": " + searchTermStr; - SwingUtilities.invokeLater(() -> { - progressIndicator.progress(progressMessage); - }); - } - - // Filtering - //limit search to currently ingested data sources - //set up a filter with 1 or more image ids OR'ed - KeywordSearchQuery keywordSearchQuery = KeywordSearchUtil.getQueryForKeyword(keyword, keywordList); - KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId()); - keywordSearchQuery.addFilter(dataSourceFilter); - - // Do the actual search - QueryResults queryResults; - try { - queryResults = keywordSearchQuery.performQuery(); - } catch (KeywordSearchModuleException | NoOpenCoreException ex) { - logger.log(Level.SEVERE, "Error performing query: " + keyword.getSearchTerm(), ex); //NON-NLS - if (usingNetBeansGUI) { - final String userMessage = Bundle.SearchRunner_query_exception_msg() + keyword.getSearchTerm(); - SwingUtilities.invokeLater(() -> { - MessageNotifyUtil.Notify.error(userMessage, ex.getCause().getMessage()); - }); - } - //no reason to continue with next query if recovery failed - //or wait for recovery to kick in and run again later - //likely case has closed and threads are being interrupted - return null; - } catch (CancellationException e) { - logger.log(Level.INFO, "Cancellation requested, exiting during keyword query: {0}", keyword.getSearchTerm()); //NON-NLS - return null; - } - - // Reduce the results of the query to only those hits we - // have not already seen. - QueryResults newResults = filterResults(queryResults); - - if (!newResults.getKeywords().isEmpty()) { - // Create blackboard artifacts - newResults.process(this, keywordList.getIngestMessages(), true, job.getJobId()); - } - } - } catch (Exception ex) { - logger.log(Level.SEVERE, String.format("Error performing keyword search for ingest job %d", job.getJobId()), ex); //NON-NLS - } finally { - if (progressIndicator != null) { - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - progressIndicator.finish(); - progressIndicator = null; - } - }); - } - // In case a thread is waiting on this worker to be done - job.searchNotify(); - } - - return null; - } - - /** - * Sync-up the updated keywords from the currently used lists in the XML - */ - private void updateKeywords() { - XmlKeywordSearchList loader = XmlKeywordSearchList.getCurrent(); - - keywords.clear(); - keywordToList.clear(); - keywordLists.clear(); - - for (String name : keywordListNames) { - KeywordList list = loader.getList(name); - keywordLists.add(list); - for (Keyword k : list.getKeywords()) { - keywords.add(k); - keywordToList.put(k, list); - } - } - } - - /** - * This method filters out all of the hits found in earlier periodic - * searches and returns only the results found by the most recent - * search. - * - * This method will only return hits for objects for which we haven't - * previously seen a hit for the keyword. - * - * @param queryResult The results returned by a keyword search. - * - * @return A unique set of hits found by the most recent search for - * objects that have not previously had a hit. The hits will be - * for the lowest numbered chunk associated with the object. - * - */ - private QueryResults filterResults(QueryResults queryResult) { - - // Create a new (empty) QueryResults object to hold the most recently - // found hits. - QueryResults newResults = new QueryResults(queryResult.getQuery()); - - // For each keyword represented in the results. - for (Keyword keyword : queryResult.getKeywords()) { - // These are all of the hits across all objects for the most recent search. - // This may well include duplicates of hits we've seen in earlier periodic searches. - List queryTermResults = queryResult.getResults(keyword); - - // Sort the hits for this keyword so that we are always - // guaranteed to return the hit for the lowest chunk. - Collections.sort(queryTermResults); - - // This will be used to build up the hits we haven't seen before - // for this keyword. - List newUniqueHits = new ArrayList<>(); - - // Get the set of object ids seen in the past by this searcher - // for the given keyword. - Set curTermResults = job.currentKeywordResults(keyword); - if (curTermResults == null) { - // We create a new empty set if we haven't seen results for - // this keyword before. - curTermResults = new HashSet<>(); - } - - // For each hit for this keyword. - for (KeywordHit hit : queryTermResults) { - if (curTermResults.contains(hit.getSolrObjectId())) { - // Skip the hit if we've already seen a hit for - // this keyword in the object. - continue; - } - - // We haven't seen the hit before so add it to list of new - // unique hits. - newUniqueHits.add(hit); - - // Add the object id to the results we've seen for this - // keyword. - curTermResults.add(hit.getSolrObjectId()); - } - - // Update the job with the list of objects for which we have - // seen hits for the current keyword. - job.addKeywordResults(keyword, curTermResults); - - // Add the new hits for the current keyword into the results - // to be returned. - newResults.addResult(keyword, newUniqueHits); - } - - return newResults; - } - } - -} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form index 4a0cadaefd..0f8800efdd 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form @@ -1,10 +1,6 @@
- - - - @@ -47,23 +43,13 @@ - - - - - - - - - - - + @@ -90,19 +76,7 @@ - - - - - - - - - - - - - + @@ -119,7 +93,7 @@ - + @@ -184,65 +158,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -253,19 +168,6 @@ - - - - - - - - - - - - - diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java index ccd1de71da..7393e3ea4d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2018 Basis Technology Corp. + * Copyright 2012-2022 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,9 +26,7 @@ import org.netbeans.spi.options.OptionsPanelController; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.IngestManager; -import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.UpdateFrequency; /** * General, not per list, keyword search configuration and status display widget @@ -53,31 +51,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen boolean ingestRunning = IngestManager.getInstance().isIngestRunning(); ingestWarningLabel.setVisible(ingestRunning); skipNSRLCheckBox.setEnabled(!ingestRunning); - setTimeSettingEnabled(!ingestRunning); - - final UpdateFrequency curFreq = KeywordSearchSettings.getUpdateFrequency(); - switch (curFreq) { - case FAST: - timeRadioButton1.setSelected(true); - break; - case AVG: - timeRadioButton2.setSelected(true); - break; - case SLOW: - timeRadioButton3.setSelected(true); - break; - case SLOWEST: - timeRadioButton4.setSelected(true); - break; - case NONE: - timeRadioButton5.setSelected(true); - break; - case DEFAULT: - default: - // default value - timeRadioButton3.setSelected(true); - break; - } } /** @@ -89,7 +62,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen // //GEN-BEGIN:initComponents private void initComponents() { - timeGroup = new javax.swing.ButtonGroup(); skipNSRLCheckBox = new javax.swing.JCheckBox(); filesIndexedLabel = new javax.swing.JLabel(); filesIndexedValue = new javax.swing.JLabel(); @@ -99,13 +71,7 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen informationLabel = new javax.swing.JLabel(); settingsSeparator = new javax.swing.JSeparator(); informationSeparator = new javax.swing.JSeparator(); - frequencyLabel = new javax.swing.JLabel(); - timeRadioButton1 = new javax.swing.JRadioButton(); - timeRadioButton2 = new javax.swing.JRadioButton(); - timeRadioButton3 = new javax.swing.JRadioButton(); - timeRadioButton4 = new javax.swing.JRadioButton(); showSnippetsCB = new javax.swing.JCheckBox(); - timeRadioButton5 = new javax.swing.JRadioButton(); ingestWarningLabel = new javax.swing.JLabel(); skipNSRLCheckBox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text")); // NOI18N @@ -128,40 +94,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen informationLabel.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.informationLabel.text")); // NOI18N - frequencyLabel.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.frequencyLabel.text")); // NOI18N - - timeRadioButton1.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.text")); // NOI18N - timeRadioButton1.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.toolTipText")); // NOI18N - timeRadioButton1.addActionListener(new java.awt.event.ActionListener() { - public void actionPerformed(java.awt.event.ActionEvent evt) { - timeRadioButton1ActionPerformed(evt); - } - }); - - timeRadioButton2.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.text")); // NOI18N - timeRadioButton2.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.toolTipText")); // NOI18N - timeRadioButton2.addActionListener(new java.awt.event.ActionListener() { - public void actionPerformed(java.awt.event.ActionEvent evt) { - timeRadioButton2ActionPerformed(evt); - } - }); - - timeRadioButton3.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text")); // NOI18N - timeRadioButton3.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText")); // NOI18N - timeRadioButton3.addActionListener(new java.awt.event.ActionListener() { - public void actionPerformed(java.awt.event.ActionEvent evt) { - timeRadioButton3ActionPerformed(evt); - } - }); - - timeRadioButton4.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1")); // NOI18N - timeRadioButton4.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.toolTipText")); // NOI18N - timeRadioButton4.addActionListener(new java.awt.event.ActionListener() { - public void actionPerformed(java.awt.event.ActionEvent evt) { - timeRadioButton4ActionPerformed(evt); - } - }); - showSnippetsCB.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.showSnippetsCB.text")); // NOI18N showSnippetsCB.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -169,14 +101,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen } }); - timeRadioButton5.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text")); // NOI18N - timeRadioButton5.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText")); // NOI18N - timeRadioButton5.addActionListener(new java.awt.event.ActionListener() { - public void actionPerformed(java.awt.event.ActionEvent evt) { - timeRadioButton5ActionPerformed(evt); - } - }); - ingestWarningLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/modules/hashdatabase/warning16.png"))); // NOI18N ingestWarningLabel.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.ingestWarningLabel.text")); // NOI18N @@ -207,19 +131,11 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen .addComponent(filesIndexedLabel) .addGap(18, 18, 18) .addComponent(filesIndexedValue)) - .addComponent(frequencyLabel) .addGroup(layout.createSequentialGroup() .addComponent(chunksLabel) .addGap(18, 18, 18) - .addComponent(chunksValLabel)) - .addGroup(layout.createSequentialGroup() - .addGap(16, 16, 16) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(timeRadioButton2) - .addComponent(timeRadioButton1) - .addComponent(timeRadioButton3) - .addComponent(timeRadioButton4) - .addComponent(timeRadioButton5)))))) + .addComponent(chunksValLabel))) + .addGap(132, 132, 132))) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) .addGroup(layout.createSequentialGroup() .addComponent(settingsLabel) @@ -241,19 +157,7 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen .addComponent(skipNSRLCheckBox) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(showSnippetsCB) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(frequencyLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(timeRadioButton1) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(timeRadioButton2) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(timeRadioButton3) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(timeRadioButton4) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(timeRadioButton5) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) .addComponent(informationLabel) .addComponent(informationSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 7, javax.swing.GroupLayout.PREFERRED_SIZE)) @@ -267,14 +171,10 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen .addComponent(chunksValLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(ingestWarningLabel) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addContainerGap(151, Short.MAX_VALUE)) ); }// //GEN-END:initComponents - private void timeRadioButton5ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton5ActionPerformed - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - }//GEN-LAST:event_timeRadioButton5ActionPerformed - private void skipNSRLCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipNSRLCheckBoxActionPerformed firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); }//GEN-LAST:event_skipNSRLCheckBoxActionPerformed @@ -283,28 +183,11 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); }//GEN-LAST:event_showSnippetsCBActionPerformed - private void timeRadioButton1ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton1ActionPerformed - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - }//GEN-LAST:event_timeRadioButton1ActionPerformed - - private void timeRadioButton2ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton2ActionPerformed - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - }//GEN-LAST:event_timeRadioButton2ActionPerformed - - private void timeRadioButton3ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton3ActionPerformed - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - }//GEN-LAST:event_timeRadioButton3ActionPerformed - - private void timeRadioButton4ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton4ActionPerformed - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - }//GEN-LAST:event_timeRadioButton4ActionPerformed - // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JLabel chunksLabel; private javax.swing.JLabel chunksValLabel; private javax.swing.JLabel filesIndexedLabel; private javax.swing.JLabel filesIndexedValue; - private javax.swing.JLabel frequencyLabel; private javax.swing.JLabel informationLabel; private javax.swing.JSeparator informationSeparator; private javax.swing.JLabel ingestWarningLabel; @@ -312,18 +195,11 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen private javax.swing.JSeparator settingsSeparator; private javax.swing.JCheckBox showSnippetsCB; private javax.swing.JCheckBox skipNSRLCheckBox; - private javax.swing.ButtonGroup timeGroup; - private javax.swing.JRadioButton timeRadioButton1; - private javax.swing.JRadioButton timeRadioButton2; - private javax.swing.JRadioButton timeRadioButton3; - private javax.swing.JRadioButton timeRadioButton4; - private javax.swing.JRadioButton timeRadioButton5; // End of variables declaration//GEN-END:variables @Override public void store() { KeywordSearchSettings.setSkipKnown(skipNSRLCheckBox.isSelected()); - KeywordSearchSettings.setUpdateFrequency(getSelectedTimeValue()); KeywordSearchSettings.setShowSnippets(showSnippetsCB.isSelected()); } @@ -332,40 +208,10 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen activateWidgets(); } - private void setTimeSettingEnabled(boolean enabled) { - timeRadioButton1.setEnabled(enabled); - timeRadioButton2.setEnabled(enabled); - timeRadioButton3.setEnabled(enabled); - timeRadioButton4.setEnabled(enabled); - timeRadioButton5.setEnabled(enabled); - frequencyLabel.setEnabled(enabled); - } - - private UpdateFrequency getSelectedTimeValue() { - if (timeRadioButton1.isSelected()) { - return UpdateFrequency.FAST; - } else if (timeRadioButton2.isSelected()) { - return UpdateFrequency.AVG; - } else if (timeRadioButton3.isSelected()) { - return UpdateFrequency.SLOW; - } else if (timeRadioButton4.isSelected()) { - return UpdateFrequency.SLOWEST; - } else if (timeRadioButton5.isSelected()) { - return UpdateFrequency.NONE; - } - return UpdateFrequency.DEFAULT; - } - @NbBundle.Messages({"KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsOCR=Enable Optical Character Recognition (OCR) (Requires Windows 64-bit)", "KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsLimitedOCR=Only process images which are over 100KB in size or extracted from a document. (Beta) (Requires Windows 64-bit)"}) private void customizeComponents() { - timeGroup.add(timeRadioButton1); - timeGroup.add(timeRadioButton2); - timeGroup.add(timeRadioButton3); - timeGroup.add(timeRadioButton4); - timeGroup.add(timeRadioButton5); - this.skipNSRLCheckBox.setSelected(KeywordSearchSettings.getSkipKnown()); try { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 3140916f5e..2d594254ea 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2021 Basis Technology Corp. + * Copyright 2011-2022 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -169,24 +169,6 @@ public final class KeywordSearchIngestModule implements FileIngestModule { EXTRACT_UTF8, ///< extract UTF8 text, true/false }; - enum UpdateFrequency { - - FAST(20), - AVG(10), - SLOW(5), - SLOWEST(1), - NONE(Integer.MAX_VALUE), - DEFAULT(5); - private final int time; - - UpdateFrequency(int time) { - this.time = time; - } - - int getTime() { - return time; - } - }; private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; @@ -195,7 +177,6 @@ public final class KeywordSearchIngestModule implements FileIngestModule { //only search images from current ingest, not images previously ingested/indexed //accessed read-only by searcher thread - private boolean startedSearching = false; private Lookup stringsExtractionContext; private final KeywordSearchJobSettings settings; private boolean initialized = false; @@ -399,16 +380,6 @@ public final class KeywordSearchIngestModule implements FileIngestModule { } indexer.indexFile(extractorOpt, abstractFile, mimeType, true); - // Start searching if it hasn't started already - if (!startedSearching) { - if (context.fileIngestIsCancelled()) { - return ProcessResult.OK; - } - List keywordListNames = settings.getNamesOfEnabledKeyWordLists(); - IngestSearchRunner.getInstance().startJob(context, keywordListNames); - startedSearching = true; - } - return ProcessResult.OK; } @@ -425,15 +396,11 @@ public final class KeywordSearchIngestModule implements FileIngestModule { } if (context.fileIngestIsCancelled()) { - logger.log(Level.INFO, "Keyword search ingest module instance {0} stopping search job due to ingest cancellation", instanceNum); //NON-NLS - IngestSearchRunner.getInstance().stopJob(jobId); + logger.log(Level.INFO, "Keyword search ingest module instance {0} stopping due to ingest cancellation", instanceNum); //NON-NLS cleanup(); return; } - // Remove from the search list and trigger final commit and final search - IngestSearchRunner.getInstance().endJob(jobId); - // We only need to post the summary msg from the last module per job if (refCounter.decrementAndGet(jobId) == 0) { try { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java index b7057a2c89..0087c7d05e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java @@ -29,7 +29,6 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; import org.sleuthkit.autopsy.coreutils.StringExtract; import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT; import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.StringsExtractOptions; -import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.UpdateFrequency; //This file contains constants and settings for KeywordSearch class KeywordSearchSettings { @@ -46,34 +45,9 @@ class KeywordSearchSettings { static final boolean LIMITED_OCR_ENABLED_DEFAULT = false; private static boolean skipKnown = true; private static final Logger logger = Logger.getLogger(KeywordSearchSettings.class.getName()); - private static UpdateFrequency UpdateFreq = UpdateFrequency.DEFAULT; private static List stringExtractScripts = new ArrayList<>(); private static Map stringExtractOptions = new HashMap<>(); - /** - * Gets the update Frequency from KeywordSearch_Options.properties - * - * @return KeywordSearchIngestModule's update frequency - */ - static UpdateFrequency getUpdateFrequency() { - if (ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, "UpdateFrequency") != null) { //NON-NLS - return UpdateFrequency.valueOf(ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, "UpdateFrequency")); //NON-NLS - } - //if it failed, return the default/last known value - logger.log(Level.WARNING, "Could not read property for UpdateFrequency, returning backup value."); //NON-NLS - return UpdateFrequency.DEFAULT; - } - - /** - * Sets the update frequency and writes to KeywordSearch_Options.properties - * - * @param freq Sets KeywordSearchIngestModule to this value. - */ - static void setUpdateFrequency(UpdateFrequency freq) { - ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, "UpdateFrequency", freq.name()); //NON-NLS - UpdateFreq = freq; - } - /** * Sets whether or not to skip adding known good files to the search during * index. @@ -243,11 +217,6 @@ class KeywordSearchSettings { logger.log(Level.INFO, "No configuration for NSRL found, generating default..."); //NON-NLS KeywordSearchSettings.setSkipKnown(true); } - //setting default Update Frequency - if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, "UpdateFrequency")) { //NON-NLS - logger.log(Level.INFO, "No configuration for Update Frequency found, generating default..."); //NON-NLS - KeywordSearchSettings.setUpdateFrequency(UpdateFrequency.DEFAULT); - } //setting default Extract UTF8 if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, StringsExtractOptions.EXTRACT_UTF8.toString())) { logger.log(Level.INFO, "No configuration for UTF8 found, generating default..."); //NON-NLS diff --git a/docs/doxygen-user/images/InterestingFiles/bomb_png.png b/docs/doxygen-user/images/InterestingFiles/bomb_png.png index 016ff5dea5..ab254f74f4 100644 Binary files a/docs/doxygen-user/images/InterestingFiles/bomb_png.png and b/docs/doxygen-user/images/InterestingFiles/bomb_png.png differ diff --git a/docs/doxygen-user/images/InterestingFiles/download_archive.png b/docs/doxygen-user/images/InterestingFiles/download_archive.png index aa457e929c..0021431e3c 100644 Binary files a/docs/doxygen-user/images/InterestingFiles/download_archive.png and b/docs/doxygen-user/images/InterestingFiles/download_archive.png differ diff --git a/docs/doxygen-user/images/InterestingFiles/if_new_rule.png b/docs/doxygen-user/images/InterestingFiles/if_new_rule.png index 826b2b1203..9164d20149 100644 Binary files a/docs/doxygen-user/images/InterestingFiles/if_new_rule.png and b/docs/doxygen-user/images/InterestingFiles/if_new_rule.png differ diff --git a/docs/doxygen-user/images/InterestingFiles/new_large_files.png b/docs/doxygen-user/images/InterestingFiles/new_large_files.png index a159f5af1f..313020c0a0 100644 Binary files a/docs/doxygen-user/images/InterestingFiles/new_large_files.png and b/docs/doxygen-user/images/InterestingFiles/new_large_files.png differ diff --git a/docs/doxygen-user/images/InterestingFiles/new_rule.png b/docs/doxygen-user/images/InterestingFiles/new_rule.png index 3385a402c0..e48460dee8 100644 Binary files a/docs/doxygen-user/images/InterestingFiles/new_rule.png and b/docs/doxygen-user/images/InterestingFiles/new_rule.png differ diff --git a/docs/doxygen-user/images/InterestingFiles/private_folder.png b/docs/doxygen-user/images/InterestingFiles/private_folder.png index 706bbeca57..f8ce78d2d4 100644 Binary files a/docs/doxygen-user/images/InterestingFiles/private_folder.png and b/docs/doxygen-user/images/InterestingFiles/private_folder.png differ diff --git a/docs/doxygen-user/ingest.dox b/docs/doxygen-user/ingest.dox index 594108ee07..9df3f724ff 100644 --- a/docs/doxygen-user/ingest.dox +++ b/docs/doxygen-user/ingest.dox @@ -50,7 +50,14 @@ The file filters panel can be opened from the ingest module selection panel or t \image html ingest-file-filters.PNG -Each filter contains one or more rules for selecting files based on a combination of file name, path, file size, and how recently the file was modified. Only one rule needs to match for the file to pass. Additionally, you can enter multiple comma-separated file extensions. All files will still be displayed in the tree view, but the ingest modules will only run on a subset. If we use the previous example and run the hash module, only files ending in .png will have their hash computed. +Each filter contains one or more rules for selecting files based on a combination of file name, path, file size, and how recently the file was modified. A file will match a rule set based on these conventions: +
    +
  • If there are no rules in a rule set, no file will match the set.
    • +
  • If there are only rules that exclude files, as long as the file doesn’t match one of the exclusive rules, it is included in the set.
    • +
  • If both inclusive rules and exclusive rules are present, a file must match one of the inclusive rules and not match any of the exclusive rules to be included in the set.
    • +
  • If there are only inclusive rules, a file must match one of the rules to be included in the set.
    • +
+Additionally, you can enter multiple comma-separated file extensions. All files will still be displayed in the tree view, but the ingest modules will only run on a subset. If we use the previous example and run the hash module, only files ending in .png will have their hash computed. \section ingest_profiles Using Ingest Profiles diff --git a/docs/doxygen-user/interesting_files.dox b/docs/doxygen-user/interesting_files.dox index 79d04ae86c..d1ea1e8d36 100644 --- a/docs/doxygen-user/interesting_files.dox +++ b/docs/doxygen-user/interesting_files.dox @@ -42,11 +42,17 @@ The buttons under the list of rules allow you to create new rules and edit or de \image html InterestingFiles/new_rule.png -The top line allows you to choose whether you want to match only files, only directories, or both. If you select directories or both, some of the condition types will be unavailable since they only apply to files. - Each rule must have at least one condition. To create conditions, check the box to the left of the condition you want to enable. The following is a description of each condition, with some full examples after.
    +
  • Rule Type - Determines if files matching this rule will be included or excluded. A rule set will match these conventions: +
      +
    • If there are no rules in a rule set, no file will match the set.
      • +
    • If there are only rules that exclude files, as long as the file doesn’t match one of the exclusive rules, it is included in the set.
      • +
    • If both inclusive rules and exclusive rules are present, a file must match one of the inclusive rules and not match any of the exclusive rules to be included in the set.
      • +
    • If there are only inclusive rules, a file must match one of the rules to be included in the set.
      • +
    +
  • Type - Choose whether you want to match only files, only directories, or both. If you select directories or both, some of the condition types will be unavailable since they only apply to files.
  • Name - Enter either the full file name or one or more extensions, and select whether this is an exact match or a substring/regex match. If substring/regex match is enabled, it will automatically add wildcards to the beginning and end of the text. If you're only matching directories, this will match the directory name. If you're using a comma-separated list of extensions, make sure the regex checkbox is disabled - the entire contents will be interpreted as one regex when the checkbox is selected. The following table shows some examples of what the different combinations can be used for. @@ -59,7 +65,7 @@ Each rule must have at least one condition. To create conditions, check the box
    Extension Onlytrue\verbatim jp \endverbatimWill match .jpg, .jpeg files, and any others with "jp" in the extensionmyImage.jpg
    -
  • Path Substring - Enter a folder name that must be part of file's path for it to be a match. If you only want to specify that a word appears somewhere in the path, use the regex option. +
  • Folder Name - Enter a folder name that must be part of file's path for it to be a match. If you only want to specify that a word appears somewhere in the path, use the regex option. @@ -91,7 +97,7 @@ This rule is looking for archives in the user download directory. It requires "U \image html InterestingFiles/download_archive.png -This is a rule that matches files with size at least 50MB that have been modified in the last week. +This is a rule that matches files with size more than 50MB that have been modified in the last week. \image html InterestingFiles/new_large_files.png diff --git a/docs/doxygen-user_fr/images/InterestingFiles/bomb_png.png b/docs/doxygen-user_fr/images/InterestingFiles/bomb_png.png index 016ff5dea5..ab254f74f4 100644 Binary files a/docs/doxygen-user_fr/images/InterestingFiles/bomb_png.png and b/docs/doxygen-user_fr/images/InterestingFiles/bomb_png.png differ diff --git a/docs/doxygen-user_fr/images/InterestingFiles/download_archive.png b/docs/doxygen-user_fr/images/InterestingFiles/download_archive.png index aa457e929c..0021431e3c 100644 Binary files a/docs/doxygen-user_fr/images/InterestingFiles/download_archive.png and b/docs/doxygen-user_fr/images/InterestingFiles/download_archive.png differ diff --git a/docs/doxygen-user_fr/images/InterestingFiles/if_new_rule.png b/docs/doxygen-user_fr/images/InterestingFiles/if_new_rule.png index 826b2b1203..9164d20149 100644 Binary files a/docs/doxygen-user_fr/images/InterestingFiles/if_new_rule.png and b/docs/doxygen-user_fr/images/InterestingFiles/if_new_rule.png differ diff --git a/docs/doxygen-user_fr/images/InterestingFiles/new_large_files.png b/docs/doxygen-user_fr/images/InterestingFiles/new_large_files.png index a159f5af1f..313020c0a0 100644 Binary files a/docs/doxygen-user_fr/images/InterestingFiles/new_large_files.png and b/docs/doxygen-user_fr/images/InterestingFiles/new_large_files.png differ diff --git a/docs/doxygen-user_fr/images/InterestingFiles/new_rule.png b/docs/doxygen-user_fr/images/InterestingFiles/new_rule.png index 3385a402c0..e48460dee8 100644 Binary files a/docs/doxygen-user_fr/images/InterestingFiles/new_rule.png and b/docs/doxygen-user_fr/images/InterestingFiles/new_rule.png differ diff --git a/docs/doxygen-user_fr/images/InterestingFiles/private_folder.png b/docs/doxygen-user_fr/images/InterestingFiles/private_folder.png index 706bbeca57..f8ce78d2d4 100644 Binary files a/docs/doxygen-user_fr/images/InterestingFiles/private_folder.png and b/docs/doxygen-user_fr/images/InterestingFiles/private_folder.png differ diff --git a/linux_macos_install_scripts/install_application.sh b/linux_macos_install_scripts/install_application.sh index b0f5781833..6a7a7cd0cb 100644 --- a/linux_macos_install_scripts/install_application.sh +++ b/linux_macos_install_scripts/install_application.sh @@ -82,7 +82,7 @@ fi echo "Setting up application at $APPLICATION_EXTRACTED_PATH..." # find unix_setup.sh in least nested path (https://stackoverflow.com/a/40039568/2375948) -UNIX_SETUP_PATH=`find $APPLICATION_EXTRACTED_PATH -name 'unix_setup.sh' -printf "%d %p\n"| sort -n | perl -pe 's/^\d+\s//;' | head -n1 | xargs -I{} dirname {}` +UNIX_SETUP_PATH=`find $APPLICATION_EXTRACTED_PATH -maxdepth 2 -name 'unix_setup.sh' | head -n1 | xargs -I{} dirname {}` pushd $UNIX_SETUP_PATH && chown -R $(whoami) . &&
    RegexTextDescriptionSample match
    false\verbatim Documents \endverbatimMatch any file that has a folder named "Documents" in its path/folder1/Documents/fileA.doc