From ee3f471e39adf47be321717c1b07c70e292e914f Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 27 Jan 2021 11:08:47 -0500 Subject: [PATCH 01/37] Initial layout --- .../keywordsearch/AdHocSearchPanel.java | 2 +- .../keywordsearch/Bundle.properties-MERGED | 2 + .../keywordsearch/ExtractAllTermsAction.java | 68 +++++++++++++++++++ .../autopsy/keywordsearch/IndexFinder.java | 7 +- .../keywordsearch/SolrSearchService.java | 4 +- 5 files changed, 75 insertions(+), 8 deletions(-) create mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java index 6190f392e8..c9aa061400 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java @@ -148,7 +148,7 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { } /** - * Get a list of data source display name. + * Get a list of data source display names. * * @return The list of data source name */ diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 02244f9477..25a07a1bbb 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -5,10 +5,12 @@ AccountsText.nextPage.exception.msg=No next page. AccountsText.previousItem.exception.msg=No previous item. AccountsText.previousPage.exception.msg=No previous page. CannotRunFileTypeDetection=Unable to run file type detection. +CTL_ExtractAllTermsAction=Extract Unique Words DropdownListSearchPanel.selected=Ad Hoc Search data source filter is selected DropdownSingleTermSearchPanel.selected=Ad Hoc Search data source filter is selected DropdownSingleTermSearchPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] DropdownSingleTermSearchPanel.warning.title=Warning +ExtractAllTermsAction.getName.text=Extract Unique Words ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait # {0} - Content name ExtractedContentPanel.SetMarkup.progress.loading=Loading text for {0} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java new file mode 100755 index 0000000000..560d018349 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java @@ -0,0 +1,68 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.awt.event.ActionEvent; +import org.openide.awt.ActionID; +import org.openide.awt.ActionReference; +import org.openide.awt.ActionRegistration; +import org.openide.util.HelpCtx; +import org.openide.util.NbBundle; +import org.openide.util.actions.CallableSystemAction; +import org.sleuthkit.autopsy.casemodule.Case; + +/** + * Action for accessing the Search Other Cases dialog. + */ +@ActionID(category = "Tools", id = "org.sleuthkit.autopsy.keywordsearch.ExtractAllTermsAction") +@ActionRegistration(displayName = "#CTL_OtherCasesSearchAction=Search All Cases", lazy = false) +@ActionReference(path = "Menu/Tools", position = 202) +@NbBundle.Messages({"CTL_ExtractAllTermsAction=Extract Unique Words"}) +public class ExtractAllTermsAction extends CallableSystemAction { + + @Override + public boolean isEnabled() { + return Case.isCaseOpen(); + } + + @Override + public void actionPerformed(ActionEvent event) { + performAction(); + } + + @Override + public void performAction() { + // ELTODO AllCasesSearchDialog dialog = new AllCasesSearchDialog(); + // ELTODO dialog.display(); + int a = 5; + } + + @NbBundle.Messages({ + "ExtractAllTermsAction.getName.text=Extract Unique Words"}) + @Override + public String getName() { + return Bundle.ExtractAllTermsAction_getName_text(); + } + + @Override + public HelpCtx getHelpCtx() { + return HelpCtx.DEFAULT_HELP; + } + +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index 99fef29810..dc9179291d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -21,10 +21,8 @@ package org.sleuthkit.autopsy.keywordsearch; import java.io.File; import java.nio.file.Paths; import java.util.List; -import java.util.logging.Level; import org.apache.commons.lang.math.NumberUtils; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.appservices.AutopsyService; /** @@ -32,7 +30,6 @@ import org.sleuthkit.autopsy.appservices.AutopsyService; */ class IndexFinder { - private static final Logger logger = Logger.getLogger(IndexFinder.class.getName()); private static final String KWS_OUTPUT_FOLDER_NAME = "keywordsearch"; private static final String KWS_DATA_FOLDER_NAME = "data"; private static final String INDEX_FOLDER_NAME = "index"; @@ -48,7 +45,7 @@ class IndexFinder { return CURRENT_SOLR_SCHEMA_VERSION; } - static Index findLatestVersionIndexDir(List allIndexes) { + static Index findLatestVersionIndex(List allIndexes) { for (Index index : allIndexes) { if (index.getSolrVersion().equals(CURRENT_SOLR_VERSION) && index.getSchemaVersion().equals(CURRENT_SOLR_SCHEMA_VERSION)) { return index; @@ -57,7 +54,7 @@ class IndexFinder { return null; } - static Index createLatestVersionIndexDir(Case theCase) throws AutopsyService.AutopsyServiceException { + static Index createLatestVersionIndex(Case theCase) throws AutopsyService.AutopsyServiceException { String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema" + CURRENT_SOLR_SCHEMA_VERSION; // new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schemaY\index" File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 31ce328cc2..7eea0c3c2f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -309,14 +309,14 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { // new case that doesn't have an existing index. create new index folder progressUnitsCompleted++; progress.progress(Bundle.SolrSearch_creatingNewIndex_msg(), progressUnitsCompleted); - currentVersionIndex = IndexFinder.createLatestVersionIndexDir(theCase); + currentVersionIndex = IndexFinder.createLatestVersionIndex(theCase); // add current index to the list of indexes that exist for this case indexes.add(currentVersionIndex); } else { // check if one of the existing indexes is for latest Solr version and schema progressUnitsCompleted++; progress.progress(Bundle.SolrSearch_checkingForLatestIndex_msg(), progressUnitsCompleted); - currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes); + currentVersionIndex = IndexFinder.findLatestVersionIndex(indexes); if (currentVersionIndex == null) { // found existing index(es) but none were for latest Solr version and schema version progressUnitsCompleted++; From 3d50d69b4a32f7489458ba8d6e07f20b5dc6d134 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 27 Jan 2021 19:03:57 -0500 Subject: [PATCH 02/37] First cut --- .../keywordsearch/ExtractAllTermsAction.java | 17 ++++++- .../autopsy/keywordsearch/Server.java | 48 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java index 560d018349..992b79131f 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java @@ -19,9 +19,12 @@ package org.sleuthkit.autopsy.keywordsearch; import java.awt.event.ActionEvent; +import java.io.IOException; +import org.apache.solr.client.solrj.SolrServerException; import org.openide.awt.ActionID; import org.openide.awt.ActionReference; import org.openide.awt.ActionRegistration; +import org.openide.util.Exceptions; import org.openide.util.HelpCtx; import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; @@ -50,7 +53,19 @@ public class ExtractAllTermsAction extends CallableSystemAction { public void performAction() { // ELTODO AllCasesSearchDialog dialog = new AllCasesSearchDialog(); // ELTODO dialog.display(); - int a = 5; + final Server server = KeywordSearch.getServer(); + Long dsID = Long.valueOf(4); + try { + server.extractAllTermsForDataSource(dsID); + } catch (IOException ex) { + Exceptions.printStackTrace(ex); + } catch (KeywordSearchModuleException ex) { + Exceptions.printStackTrace(ex); + } catch (NoOpenCoreException ex) { + Exceptions.printStackTrace(ex); + } catch (SolrServerException ex) { + Exceptions.printStackTrace(ex); + } } @NbBundle.Messages({ diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index a27eb4db78..39c47db9bb 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -65,8 +65,10 @@ import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.response.CoreAdminResponse; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException; +import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.TermsResponse; +import org.apache.solr.client.solrj.response.TermsResponse.Term; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; @@ -1766,6 +1768,25 @@ public class Server { currentCoreLock.writeLock().unlock(); } } + + /** + * Extract all unique terms/words for a given data source. + * + * @param dataSourceId to process + * + * @throws NoOpenCoreException + */ + void extractAllTermsForDataSource(Long dataSourceId) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException { + try { + currentCoreLock.writeLock().lock(); + if (null == currentCollection) { + throw new NoOpenCoreException(); + } + currentCollection.extractAllTermsForDataSource(dataSourceId); + } finally { + currentCoreLock.writeLock().unlock(); + } + } /** * Get the text contents of the given file as stored in SOLR. @@ -2144,6 +2165,33 @@ public class Server { queryClient.deleteByQuery(deleteQuery); } + + private void extractAllTermsForDataSource(Long dsObjId) throws IOException, SolrServerException { + String dataSourceId = Long.toString(dsObjId); + + SolrQuery query = new SolrQuery(); + query.setRequestHandler("/terms"); + query.setTerms(true); + query.setTermsLimit(20); + //query.setTermsLower("s"); + //query.setTermsPrefix("s"); + query.addTermsField("text"); + query.setTermsMinCount(1); + + query.addFilterQuery("image_id:" + dataSourceId); + + QueryRequest request = new QueryRequest(query); + List terms = request.process(queryClient).getTermsResponse().getTerms("text"); + + if (terms == null || terms.isEmpty()) { + logger.log(Level.WARNING, "No unique terms/words returned for data source ID: " + dataSourceId); //NON-NLS + return; + } + + Term term = terms.get(0); + String word = term.getTerm(); + long frequency = term.getFrequency(); + } /** * Add a Solr document for indexing. Documents get batched instead of From bd2add31add4d4027e63fbc97767fb59537ec1f4 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 29 Jan 2021 16:37:26 -0500 Subject: [PATCH 03/37] Got something to work --- .../keywordsearch/ExtractAllTermsAction.java | 8 +- .../autopsy/keywordsearch/Server.java | 125 ++++++++++++++++-- 2 files changed, 118 insertions(+), 15 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java index 992b79131f..a3e693b578 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java @@ -57,13 +57,7 @@ public class ExtractAllTermsAction extends CallableSystemAction { Long dsID = Long.valueOf(4); try { server.extractAllTermsForDataSource(dsID); - } catch (IOException ex) { - Exceptions.printStackTrace(ex); - } catch (KeywordSearchModuleException ex) { - Exceptions.printStackTrace(ex); - } catch (NoOpenCoreException ex) { - Exceptions.printStackTrace(ex); - } catch (SolrServerException ex) { + } catch (Exception ex) { Exceptions.printStackTrace(ex); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 39c47db9bb..524d519579 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -35,14 +35,17 @@ import java.net.ServerSocket; import java.net.SocketException; import java.nio.charset.Charset; import java.nio.file.Files; +import java.nio.file.OpenOption; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -51,11 +54,13 @@ import java.util.logging.Level; import javax.swing.AbstractAction; import org.apache.commons.io.FileUtils; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; @@ -81,6 +86,7 @@ import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.casemodule.CaseMetadata; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.coreutils.Logger; @@ -1776,7 +1782,7 @@ public class Server { * * @throws NoOpenCoreException */ - void extractAllTermsForDataSource(Long dataSourceId) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException { + void extractAllTermsForDataSource(Long dataSourceId) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException, NoCurrentCaseException { try { currentCoreLock.writeLock().lock(); if (null == currentCollection) { @@ -2166,31 +2172,134 @@ public class Server { queryClient.deleteByQuery(deleteQuery); } - private void extractAllTermsForDataSource(Long dsObjId) throws IOException, SolrServerException { + private void extractAllTermsForDataSource(Long dsObjId) throws IOException, SolrServerException, NoCurrentCaseException, KeywordSearchModuleException { String dataSourceId = Long.toString(dsObjId); + + int numTerms = 400000; + int termStep = 1000; SolrQuery query = new SolrQuery(); query.setRequestHandler("/terms"); query.setTerms(true); - query.setTermsLimit(20); + query.setTermsLimit(numTerms); + query.setTermsSortString("index"); + //query.setSort(SortClause.asc("id")); //query.setTermsLower("s"); //query.setTermsPrefix("s"); query.addTermsField("text"); - query.setTermsMinCount(1); + query.setTermsMinCount(0); query.addFilterQuery("image_id:" + dataSourceId); + + // degbug + String sort1 = query.getFacetSortString(); + String sort2 = query.getSortField(); + List sort3 = query.getSorts(); + String sort4 = query.getTermsSortString(); QueryRequest request = new QueryRequest(query); - List terms = request.process(queryClient).getTermsResponse().getTerms("text"); + TermsResponse response = request.process(queryClient).getTermsResponse(); + List terms = response.getTerms("text"); if (terms == null || terms.isEmpty()) { logger.log(Level.WARNING, "No unique terms/words returned for data source ID: " + dataSourceId); //NON-NLS return; } - Term term = terms.get(0); - String word = term.getTerm(); - long frequency = term.getFrequency(); + //Term term = terms.get(0); + //String word = term.getTerm(); + //long frequency = term.getFrequency(); + + // Write the server data to a file + Case currentCase = Case.getCurrentCaseThrows(); + File caseDirectoryPath = new File(currentCase.getOutputDirectory()); + Path serverFile = Paths.get(caseDirectoryPath.toString(), "terms_"+numTerms+"_whole.txt"); //NON-NLS + Files.deleteIfExists(serverFile); + + List listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList()); + + OpenOption[] options = new OpenOption[] { java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND }; + Files.write(serverFile, listTerms, options); + /*for (Term term : terms) { + try { + Files.write(serverFile, term.getTerm().getBytes(), options); + Files.write(serverFile, "\n".getBytes(), options); + } catch (IOException ex) { + throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex); //NON-NLS + } + }*/ + + // repeat the same thing but stepping through the terms + String firstTerm = ""; + Path serverFileIterated = Paths.get(caseDirectoryPath.toString(), "terms_"+numTerms+"_iterated.txt"); //NON-NLS + Files.deleteIfExists(serverFileIterated); + Map termsMap = new HashMap<>(); + for (int step = 0; step < numTerms; step += termStep) { + query = new SolrQuery(); + query.setRequestHandler("/terms"); + query.setTerms(true); + query.setTermsLimit(termStep); + query.setTermsLower(firstTerm); + query.setTermsLowerInclusive(false); + query.setTermsSortString("index"); + //query.setSort(SortClause.desc("index")); + //query.setTermsPrefix(firstTerm); + query.addTermsField("text"); + query.setTermsMinCount(0); + + query.addFilterQuery("image_id:" + dataSourceId); + + request = new QueryRequest(query); + response = request.process(queryClient).getTermsResponse(); + terms = response.getTerms("text"); + + if (terms == null || terms.isEmpty()) { + logger.log(Level.WARNING, "No unique terms/words returned for data source ID: " + dataSourceId); //NON-NLS + break; + } + + // set the first term for the next query + firstTerm = terms.get(terms.size()-1).getTerm(); + + //Term term = terms.get(0); + //String word = term.getTerm(); + //long frequency = term.getFrequency(); + // Write the server data to a file + serverFile = Paths.get(caseDirectoryPath.toString(), "terms_step" + step + "_iterated.txt"); //NON-NLS + Files.deleteIfExists(serverFile); + listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList()); + Files.write(serverFile, listTerms, options); + Files.write(serverFileIterated, listTerms, options); + + for (Term term : terms) { + Integer oldValue = termsMap.get(term.getTerm()); + if (oldValue == null) { + termsMap.put(term.getTerm(), 1); + } else { + termsMap.put(term.getTerm(), oldValue + 1); + } + //termsMap.put(term.getTerm(), 1); + /*try { + Files.write(serverFile, term.getTerm().getBytes(), options); + Files.write(serverFile, "\n".getBytes(), options); + + Files.write(serverFileIterated, term.getTerm().getBytes(), options); + Files.write(serverFileIterated, "\n".getBytes(), options); + } catch (IOException ex) { + throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex); //NON-NLS + }*/ + } + } + + // print the hash map + serverFile = Paths.get(caseDirectoryPath.toString(), "terms_step_hashMap.txt"); //NON-NLS + for (String key : termsMap.keySet()) { + Integer value = termsMap.get(key); + Files.write(serverFile, key.getBytes(), options); + //Files.write(serverFile, " ".getBytes(), options); + //Files.write(serverFile, value, options); + Files.write(serverFile, "\n".getBytes(), options); + } } /** From 02439c737e508822cf4b5ddab16bc4b2e9e2778e Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 3 Feb 2021 15:57:21 -0500 Subject: [PATCH 04/37] Trying things out --- .../keywordsearch/ExtractAllTermsAction.java | 2 +- .../autopsy/keywordsearch/Server.java | 52 +------------------ 2 files changed, 3 insertions(+), 51 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java index a3e693b578..b7bc000d58 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java @@ -34,7 +34,7 @@ import org.sleuthkit.autopsy.casemodule.Case; * Action for accessing the Search Other Cases dialog. */ @ActionID(category = "Tools", id = "org.sleuthkit.autopsy.keywordsearch.ExtractAllTermsAction") -@ActionRegistration(displayName = "#CTL_OtherCasesSearchAction=Search All Cases", lazy = false) +@ActionRegistration(displayName = "#CTL_OtherExtractAllTermsAction=Extract Unique Words", lazy = false) @ActionReference(path = "Menu/Tools", position = 202) @NbBundle.Messages({"CTL_ExtractAllTermsAction=Extract Unique Words"}) public class ExtractAllTermsAction extends CallableSystemAction { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 524d519579..019f42e3de 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -2183,19 +2183,10 @@ public class Server { query.setTerms(true); query.setTermsLimit(numTerms); query.setTermsSortString("index"); - //query.setSort(SortClause.asc("id")); - //query.setTermsLower("s"); - //query.setTermsPrefix("s"); query.addTermsField("text"); query.setTermsMinCount(0); - query.addFilterQuery("image_id:" + dataSourceId); - - // degbug - String sort1 = query.getFacetSortString(); - String sort2 = query.getSortField(); - List sort3 = query.getSorts(); - String sort4 = query.getTermsSortString(); + // ELTODO query.addFilterQuery("image_id:" + dataSourceId); QueryRequest request = new QueryRequest(query); TermsResponse response = request.process(queryClient).getTermsResponse(); @@ -2233,7 +2224,6 @@ public class Server { String firstTerm = ""; Path serverFileIterated = Paths.get(caseDirectoryPath.toString(), "terms_"+numTerms+"_iterated.txt"); //NON-NLS Files.deleteIfExists(serverFileIterated); - Map termsMap = new HashMap<>(); for (int step = 0; step < numTerms; step += termStep) { query = new SolrQuery(); query.setRequestHandler("/terms"); @@ -2242,12 +2232,10 @@ public class Server { query.setTermsLower(firstTerm); query.setTermsLowerInclusive(false); query.setTermsSortString("index"); - //query.setSort(SortClause.desc("index")); - //query.setTermsPrefix(firstTerm); query.addTermsField("text"); query.setTermsMinCount(0); - query.addFilterQuery("image_id:" + dataSourceId); + // ELTODO query.addFilterQuery("image_id:" + dataSourceId); request = new QueryRequest(query); response = request.process(queryClient).getTermsResponse(); @@ -2261,44 +2249,8 @@ public class Server { // set the first term for the next query firstTerm = terms.get(terms.size()-1).getTerm(); - //Term term = terms.get(0); - //String word = term.getTerm(); - //long frequency = term.getFrequency(); - // Write the server data to a file - serverFile = Paths.get(caseDirectoryPath.toString(), "terms_step" + step + "_iterated.txt"); //NON-NLS - Files.deleteIfExists(serverFile); listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList()); - Files.write(serverFile, listTerms, options); Files.write(serverFileIterated, listTerms, options); - - for (Term term : terms) { - Integer oldValue = termsMap.get(term.getTerm()); - if (oldValue == null) { - termsMap.put(term.getTerm(), 1); - } else { - termsMap.put(term.getTerm(), oldValue + 1); - } - //termsMap.put(term.getTerm(), 1); - /*try { - Files.write(serverFile, term.getTerm().getBytes(), options); - Files.write(serverFile, "\n".getBytes(), options); - - Files.write(serverFileIterated, term.getTerm().getBytes(), options); - Files.write(serverFileIterated, "\n".getBytes(), options); - } catch (IOException ex) { - throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex); //NON-NLS - }*/ - } - } - - // print the hash map - serverFile = Paths.get(caseDirectoryPath.toString(), "terms_step_hashMap.txt"); //NON-NLS - for (String key : termsMap.keySet()) { - Integer value = termsMap.get(key); - Files.write(serverFile, key.getBytes(), options); - //Files.write(serverFile, " ".getBytes(), options); - //Files.write(serverFile, value, options); - Files.write(serverFile, "\n".getBytes(), options); } } From 398b30741a67385218b9679f4656af001652965a Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 12 Feb 2021 15:41:43 -0500 Subject: [PATCH 05/37] First cut at UI --- .../autopsy/keywordsearch/Bundle.properties | 3 +- .../keywordsearch/Bundle.properties-MERGED | 6 +- .../keywordsearch/Bundle_ja.properties | 2 +- .../keywordsearch/ExtractAllTermsAction.java | 14 +- .../keywordsearch/TermsExtractionDialog.form | 101 +++++ .../keywordsearch/TermsExtractionDialog.java | 377 ++++++++++++++++++ 6 files changed, 487 insertions(+), 16 deletions(-) create mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form create mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index d2ab5f76c7..9ad114de47 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -23,7 +23,7 @@ KeywordSearchEditListPanel.emptyKeyword.text=Empty keyword KeywordSearchEditListPanel.errorAddingKeywords.text=Error adding keyword(s) KeywordSearchListsManagementPanel.newListButton.text=New List KeywordSearchListsManagementPanel.importButton.text=Import List -KeywordSearchListsViewerPanel.searchAddButton.text=Search +KeywordSearchListsViewerPanel.searchAddButton.text=Extract KeywordSearchListsViewerPanel.manageListsButton.text=Manage Lists KeywordSearchListsViewerPanel.ingestIndexLabel.text=Files Indexed: KeywordSearchEditListPanel.ingestMessagesCheckbox.text=Send ingest inbox messages for each hit @@ -320,3 +320,4 @@ ExtractedContentPanel.pageOfLabel.text=of ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) +TermsExtractionDialog.selectDataSourceLabel.text=Select Data Source: diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 25a07a1bbb..aef0bc570a 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -62,7 +62,7 @@ KeywordSearchEditListPanel.emptyKeyword.text=Empty keyword KeywordSearchEditListPanel.errorAddingKeywords.text=Error adding keyword(s) KeywordSearchListsManagementPanel.newListButton.text=New List KeywordSearchListsManagementPanel.importButton.text=Import List -KeywordSearchListsViewerPanel.searchAddButton.text=Search +KeywordSearchListsViewerPanel.searchAddButton.text=Extract KeywordSearchListsViewerPanel.manageListsButton.text=Manage Lists KeywordSearchListsViewerPanel.ingestIndexLabel.text=Files Indexed: KeywordSearchEditListPanel.ingestMessagesCheckbox.text=Send ingest inbox messages for each hit @@ -234,7 +234,7 @@ Server.query.exception.msg=Error running query: {0} Server.query2.exception.msg=Error running query: {0} Server.queryTerms.exception.msg=Error running terms query: {0} Server.connect.exception.msg=Failed to connect to Solr server: {0} -Server.openCore.exception.msg=Keyword search service not yet running +Server.openCore.exception.msg=Local keyword search service not yet running Server.openCore.exception.cantOpen.msg=Could not create or open index Server.openCore.exception.noIndexDir.msg=Index directory could not be created or is missing Server.request.exception.exception.msg=Could not issue Solr request @@ -389,6 +389,8 @@ ExtractedContentPanel.pageOfLabel.text=of ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) +TermsExtractionDialog.selectDataSourceLabel.text=Select Data Source: +TermsExtractionPanel.selected=Ad Hoc Search data source filter is selected TextZoomPanel.zoomInButton.text= TextZoomPanel.zoomOutButton.text= TextZoomPanel.zoomResetButton.text=Reset diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties index 2f50e62095..06ee12edc7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties @@ -398,4 +398,4 @@ ExtractedContentPanel.pageOfLabel.text=/ ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=\u30da\u30fc\u30b8: TextZoomPanel.zoomResetButton.text=\u30ea\u30bb\u30c3\u30c8 - +TermsExtractionDialog.selectDataSourceLabel.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb: diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java index b7bc000d58..c68061f8d5 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java @@ -19,12 +19,9 @@ package org.sleuthkit.autopsy.keywordsearch; import java.awt.event.ActionEvent; -import java.io.IOException; -import org.apache.solr.client.solrj.SolrServerException; import org.openide.awt.ActionID; import org.openide.awt.ActionReference; import org.openide.awt.ActionRegistration; -import org.openide.util.Exceptions; import org.openide.util.HelpCtx; import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; @@ -51,15 +48,8 @@ public class ExtractAllTermsAction extends CallableSystemAction { @Override public void performAction() { - // ELTODO AllCasesSearchDialog dialog = new AllCasesSearchDialog(); - // ELTODO dialog.display(); - final Server server = KeywordSearch.getServer(); - Long dsID = Long.valueOf(4); - try { - server.extractAllTermsForDataSource(dsID); - } catch (Exception ex) { - Exceptions.printStackTrace(ex); - } + TermsExtractionDialog listsPanel = TermsExtractionDialog.getDefault(); + listsPanel.display(); } @NbBundle.Messages({ diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form new file mode 100755 index 0000000000..302f985718 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form @@ -0,0 +1,101 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java new file mode 100755 index 0000000000..36f5c44d91 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java @@ -0,0 +1,377 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.awt.Cursor; +import java.awt.EventQueue; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import javax.swing.DefaultListModel; +import javax.swing.ListSelectionModel; +import javax.swing.event.ListSelectionEvent; +import org.openide.util.Exceptions; +import org.openide.util.NbBundle; +import org.openide.windows.WindowManager; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.IngestManager; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Viewer panel widget for keyword lists that is used in the ingest config and + * options area. + */ +@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives +class TermsExtractionDialog extends javax.swing.JDialog { + + private static final Logger logger = Logger.getLogger(TermsExtractionDialog.class.getName()); + + // ELTODO + private final String keywordSearchErrorDialogHeader = org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.dialogErrorHeader"); + private static TermsExtractionDialog instance; + private ActionListener searchAddListener; + private boolean ingestRunning; + private final Map dataSourceMap = new HashMap<>(); + private List dataSources = new ArrayList<>(); + private final DefaultListModel dataSourceListModel = new DefaultListModel<>(); + + /** + * Creates new form TermsExtractionPanel + */ + private TermsExtractionDialog() { + initComponents(); + customizeComponents(); + dataSourceList.setModel(getDataSourceListModel()); + + dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { + firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); // ELTODO + }); + } + + static synchronized TermsExtractionDialog getDefault() { + if (instance == null) { + instance = new TermsExtractionDialog(); + } + return instance; + } + + /** + * Display the Search Other Cases dialog. + */ + void display() { + updateComponents(); + this.setLocationRelativeTo(WindowManager.getDefault().getMainWindow()); + setVisible(true); + } + + private void customizeComponents() { + + ingestRunning = IngestManager.getInstance().isIngestRunning(); + updateComponents(); + + IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent evt) { + Object source = evt.getSource(); + if (source instanceof String && ((String) source).equals("LOCAL")) { //NON-NLS + EventQueue.invokeLater(() -> { + ingestRunning = IngestManager.getInstance().isIngestRunning(); + updateComponents(); + }); + } + } + }); + + searchAddListener = new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + if (ingestRunning) { +// ELTODO IngestSearchRunner.getInstance().addKeywordListsToAllJobs(listsTableModel.getSelectedLists()); + logger.log(Level.INFO, "Ingest is running"); //NON-NLS + } else { + searchAction(e); + } + } + }; + + extractTermsButton.addActionListener(searchAddListener); + } + + private void updateComponents() { + ingestRunning = IngestManager.getInstance().isIngestRunning(); + if (ingestRunning) { // ELTODO + extractTermsButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestTitle")); + extractTermsButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestMsg")); + + } else { + extractTermsButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.searchIngestTitle")); + extractTermsButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIdxSearchMsg")); + } + + try { + updateDataSourceListModel(); + } catch (Exception ex) { + // ELTODO + logger.log(Level.SEVERE, "Unable to populate list of data sources", ex); //NON-NLS + } + } + + /** + * Get a list of data source display names. + * + * @return The list of data source name + */ + synchronized List getDataSourceArray() { + List dsList = new ArrayList<>(); + Collections.sort(this.dataSources, (DataSource ds1, DataSource ds2) -> ds1.getName().compareTo(ds2.getName())); + for (DataSource ds : dataSources) { + String dsName = ds.getName(); + File dataSourceFullName = new File(dsName); + String displayName = dataSourceFullName.getName(); + dataSourceMap.put(ds.getId(), displayName); + dsList.add(displayName); + } + return dsList; + } + + /** + * Set dataSources + * @param dataSources A list of DataSource + */ + synchronized void setDataSources(List dataSources) { + this.dataSources = dataSources; + } + + /** + * Get dataSourceMap with object id and data source display name. + * + * @return The list of data source name + */ + Map getDataSourceMap() { + return dataSourceMap; + } + + /** + * Get a list of DataSourceListModel + * @return A list of DataSourceListModel + */ + final DefaultListModel getDataSourceListModel() { + return dataSourceListModel; + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + extractTermsButton = new javax.swing.JButton(); + jScrollPane1 = new javax.swing.JScrollPane(); + dataSourceList = new javax.swing.JList<>(); + selectDataSourceLabel = new javax.swing.JLabel(); + + setSize(new java.awt.Dimension(500, 200)); + + extractTermsButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/keywordsearch/search-icon.png"))); // NOI18N + extractTermsButton.setText(org.openide.util.NbBundle.getMessage(TermsExtractionDialog.class, "KeywordSearchListsViewerPanel.searchAddButton.text")); // NOI18N + extractTermsButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + extractTermsButtonActionPerformed(evt); + } + }); + + dataSourceList.setMinimumSize(new java.awt.Dimension(0, 200)); + jScrollPane1.setViewportView(dataSourceList); + + selectDataSourceLabel.setFont(selectDataSourceLabel.getFont().deriveFont(selectDataSourceLabel.getFont().getSize()-1f)); + selectDataSourceLabel.setText(org.openide.util.NbBundle.getMessage(TermsExtractionDialog.class, "TermsExtractionDialog.selectDataSourceLabel.text")); // NOI18N + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); + getContentPane().setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(extractTermsButton) + .addComponent(selectDataSourceLabel)) + .addGap(0, 0, Short.MAX_VALUE)) + .addGroup(layout.createSequentialGroup() + .addComponent(jScrollPane1) + .addContainerGap()) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGap(15, 15, 15) + .addComponent(selectDataSourceLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGap(18, 18, 18) + .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 65, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(extractTermsButton) + .addContainerGap(20, Short.MAX_VALUE)) + ); + }// //GEN-END:initComponents + + private void extractTermsButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_extractTermsButtonActionPerformed + }//GEN-LAST:event_extractTermsButtonActionPerformed + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JList dataSourceList; + private javax.swing.JButton extractTermsButton; + private javax.swing.JScrollPane jScrollPane1; + private javax.swing.JLabel selectDataSourceLabel; + // End of variables declaration//GEN-END:variables + + private void searchAction(ActionEvent e) { + setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + + try { + search(); + } finally { + setCursor(null); + } + } + + /** + * Performs the search using the selected keywords. Creates a + * DataResultTopComponent with the results. + * + * @param saveResults Flag whether to save search results as KWS artifacts. + */ + void search() { + boolean isIngestRunning = IngestManager.getInstance().isIngestRunning(); + + int filesIndexed = 0; + try { // see if another node added any indexed files + filesIndexed = KeywordSearch.getServer().queryNumIndexedFiles(); + } catch (KeywordSearchModuleException | NoOpenCoreException ignored) { + } + + if (filesIndexed == 0) { + // ELTODO + if (isIngestRunning) { + KeywordSearchUtil.displayDialog(keywordSearchErrorDialogHeader, NbBundle.getMessage(this.getClass(), + "AbstractKeywordSearchPerformer.search.noFilesInIdxMsg", + KeywordSearchSettings.getUpdateFrequency().getTime()), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); + } else { + KeywordSearchUtil.displayDialog(keywordSearchErrorDialogHeader, NbBundle.getMessage(this.getClass(), + "AbstractKeywordSearchPerformer.search.noFilesIdxdMsg"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); + } + return; + } + + //check if keyword search module ingest is running (indexing, etc) + if (isIngestRunning) { + if (KeywordSearchUtil.displayConfirmDialog(org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.searchIngestInProgressTitle"), + NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.ingestInProgressBody"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN) == false) { + return; + } + } + + final Server server = KeywordSearch.getServer(); + Long dsID = Long.valueOf(4); + Set selectedDs = getDataSourcesSelected(); + try { + server.extractAllTermsForDataSource(selectedDs.iterator().next()); //ELTODO check if not empty + } catch (Exception ex) { + Exceptions.printStackTrace(ex); + } + } + + void addSearchButtonActionListener(ActionListener al) { + extractTermsButton.addActionListener(al); + } + + /** + * Get a set of data source object ids that are selected. + * @return A set of selected object ids. + */ + Set getDataSourcesSelected() { + Set dataSourceObjIdSet = new HashSet<>(); + for (Long key : getDataSourceMap().keySet()) { + String value = getDataSourceMap().get(key); + for (String dataSource : this.dataSourceList.getSelectedValuesList()) { + if (value.equals(dataSource)) { + dataSourceObjIdSet.add(key); + } + } + } + return dataSourceObjIdSet; + } + + /** + * Update the dataSourceListModel + */ + @NbBundle.Messages({"TermsExtractionPanel.selected=Ad Hoc Search data source filter is selected"}) + void updateDataSourceListModel() throws NoCurrentCaseException, TskCoreException { + dataSources = getDataSourceList(); + getDataSourceListModel().removeAllElements(); + for (String dsName : getDataSourceArray()) { + getDataSourceListModel().addElement(dsName); + } + setComponentsEnabled(); + firePropertyChange(Bundle.TermsExtractionPanel_selected(), null, null); // ELTODO + } + + /** + * Get a list of DataSource from case database + * @return A list of DataSource + * @throws NoCurrentCaseException + * @throws TskCoreException + */ + private synchronized List getDataSourceList() throws NoCurrentCaseException, TskCoreException { + Case openCase = Case.getCurrentCaseThrows(); + return openCase.getSleuthkitCase().getDataSources(); + } + + /** + * Set the dataSourceList enabled if there are data sources to select + */ + private void setComponentsEnabled() { + + this.dataSourceList.setEnabled(false); + extractTermsButton.setEnabled(false); + if (getDataSourceListModel().size() > 1) { + this.dataSourceList.setEnabled(true); + this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + dataSourceList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION); // do not allow multiselect + extractTermsButton.setEnabled(true); + } else if (getDataSourceListModel().size() == 1) { + this.dataSourceList.setEnabled(false); + this.dataSourceList.setSelectionInterval(0, 0); + extractTermsButton.setEnabled(true); + } + } +} From 7de5f19e832ff0d57d03fc33b9ca7eee5cf7a52e Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 12 Feb 2021 17:05:25 -0500 Subject: [PATCH 06/37] More UI work --- .../autopsy/keywordsearch/Bundle.properties | 1 + .../keywordsearch/Bundle.properties-MERGED | 12 +--- .../keywordsearch/Bundle_ja.properties | 1 + .../keywordsearch/TermsExtractionDialog.form | 25 ++++++- .../keywordsearch/TermsExtractionDialog.java | 66 ++++++++++++------- 5 files changed, 66 insertions(+), 39 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 9ad114de47..8a0054c80d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -321,3 +321,4 @@ ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) TermsExtractionDialog.selectDataSourceLabel.text=Select Data Source: +TermsExtractionDialog.statusLabel.text= diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index aef0bc570a..f4c0be0948 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -12,7 +12,6 @@ DropdownSingleTermSearchPanel.warning.text=Boundary characters ^ and $ do not ma DropdownSingleTermSearchPanel.warning.title=Warning ExtractAllTermsAction.getName.text=Extract Unique Words ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait -# {0} - Content name ExtractedContentPanel.SetMarkup.progress.loading=Loading text for {0} GlobalEditListPanel.editKeyword.title=Edit Keyword GlobalEditListPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] @@ -23,16 +22,12 @@ IndexedText.warningMessage.noTextAvailable=No in KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsOCR=Enable Optical Character Recognition (OCR) (Requires Windows 64-bit) KeywordSearchGlobalSettingsPanel.Title=Global Keyword Search Settings KeywordSearchIngestModule.init.badInitMsg=Keyword search server was not properly initialized, cannot run keyword search ingest. -# {0} - Reason for not connecting to Solr KeywordSearchIngestModule.init.exception.errConnToSolr.msg=Error connecting to SOLR server: {0}. -# {0} - Reason for not starting Solr KeywordSearchIngestModule.init.tryStopSolrMsg={0}
Please try stopping Java Solr processes if any exist and restart the application. KeywordSearchIngestModule.metadataTitle=METADATA KeywordSearchIngestModule.noOpenCase.errMsg=No open case available. KeywordSearchIngestModule.startUp.noOpenCore.msg=The index could not be opened or does not exist. -# {0} - schema version number KeywordSearchIngestModule.startupException.indexSchemaNotSupported=Adding text no longer supported for schema version {0} of the text index. -# {0} - Solr version number KeywordSearchIngestModule.startupException.indexSolrVersionNotSupported=Adding text no longer supported for Solr version {0} of the text index. KeywordSearchIngestModule.startupMessage.failedToGetIndexSchema=Failed to get schema version for text index. KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found. @@ -215,7 +210,6 @@ KeywordSearchSettings.propertiesNSRL.text={0}_NSRL KeywordSearchSettings.propertiesScripts.text={0}_Scripts NoOpenCoreException.err.noOpenSorlCore.msg=No currently open Solr core. SearchRunner.query.exception.msg=Error performing query: -# {0} - colelction name Server.deleteCore.exception.msg=Failed to delete Solr colelction {0} Server.exceptionMessage.unableToBackupCollection=Unable to backup Solr collection Server.exceptionMessage.unableToCreateCollection=Unable to create Solr collection @@ -342,20 +336,15 @@ SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr an SolrSearch.complete.msg=Text index successfully opened SolrSearch.creatingNewIndex.msg=Creating new text index SolrSearch.findingIndexes.msg=Looking for existing text index directories -# {0} - futureVersion -# {1} - currentVersion SolrSearch.futureIndexVersion.msg=The text index for the case is for Solr {0}. This version of Autopsy is compatible with Solr {1}. SolrSearch.indentifyingIndex.msg=Identifying text index to use SolrSearch.lookingForMetadata.msg=Looking for text index metadata file SolrSearch.openCore.msg=Opening text index. For large cases this may take several minutes. SolrSearch.readingIndexes.msg=Reading text index metadata file SolrSearch.unableToFindIndex.msg=Unable to find index that can be used for this case -# {0} - index folder path SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0} SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case -# {0} - case directory SolrSearchService.exceptionMessage.noIndexMetadata=Unable to create IndexMetaData from case directory: {0} -# {0} - collection name SolrSearchService.exceptionMessage.unableToDeleteCollection=Unable to delete collection {0} SolrSearchService.indexingError=Unable to index blackboard artifact. SolrSearchService.ServiceName=Solr Keyword Search Service @@ -390,6 +379,7 @@ ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) TermsExtractionDialog.selectDataSourceLabel.text=Select Data Source: +TermsExtractionDialog.statusLabel.text= TermsExtractionPanel.selected=Ad Hoc Search data source filter is selected TextZoomPanel.zoomInButton.text= TextZoomPanel.zoomOutButton.text= diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties index 06ee12edc7..cf7f6f92cb 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties @@ -399,3 +399,4 @@ ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=\u30da\u30fc\u30b8: TextZoomPanel.zoomResetButton.text=\u30ea\u30bb\u30c3\u30c8 TermsExtractionDialog.selectDataSourceLabel.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb: +TermsExtractionDialog.statusLabel.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb: diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form index 302f985718..c398b11e0a 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form @@ -26,13 +26,17 @@ - + + + + + - + @@ -45,7 +49,10 @@ - + + + + @@ -97,5 +104,17 @@ + + + + + + + + + + + + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java index 36f5c44d91..52908682cc 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java @@ -51,12 +51,14 @@ import org.sleuthkit.datamodel.TskCoreException; * options area. */ @SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives +@NbBundle.Messages({ + "TermsExtractionDialog.dialogErrorHeader=Error Extracting Unique Words"}) class TermsExtractionDialog extends javax.swing.JDialog { private static final Logger logger = Logger.getLogger(TermsExtractionDialog.class.getName()); // ELTODO - private final String keywordSearchErrorDialogHeader = org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.dialogErrorHeader"); + private final String errorDialogHeader = Bundle.TermsExtractionDialog_dialogErrorHeader(); private static TermsExtractionDialog instance; private ActionListener searchAddListener; private boolean ingestRunning; @@ -73,7 +75,7 @@ class TermsExtractionDialog extends javax.swing.JDialog { dataSourceList.setModel(getDataSourceListModel()); dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { - firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); // ELTODO + firePropertyChange(Bundle.TermsExtractionPanel_selected(), null, null); // ELTODO }); } @@ -126,21 +128,16 @@ class TermsExtractionDialog extends javax.swing.JDialog { extractTermsButton.addActionListener(searchAddListener); } + @NbBundle.Messages({ + "TermsExtractionDialog.unableToGetDataSources.error=Unable to populate list of data sources"}) private void updateComponents() { + statusLabel.setText(""); ingestRunning = IngestManager.getInstance().isIngestRunning(); - if (ingestRunning) { // ELTODO - extractTermsButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestTitle")); - extractTermsButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestMsg")); - - } else { - extractTermsButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.searchIngestTitle")); - extractTermsButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIdxSearchMsg")); - } try { updateDataSourceListModel(); - } catch (Exception ex) { - // ELTODO + } catch (NoCurrentCaseException | TskCoreException ex) { + statusLabel.setText(Bundle.TermsExtractionDialog_unableToGetDataSources_error()); logger.log(Level.SEVERE, "Unable to populate list of data sources", ex); //NON-NLS } } @@ -201,6 +198,7 @@ class TermsExtractionDialog extends javax.swing.JDialog { jScrollPane1 = new javax.swing.JScrollPane(); dataSourceList = new javax.swing.JList<>(); selectDataSourceLabel = new javax.swing.JLabel(); + statusLabel = new javax.swing.JLabel(); setSize(new java.awt.Dimension(500, 200)); @@ -218,17 +216,23 @@ class TermsExtractionDialog extends javax.swing.JDialog { selectDataSourceLabel.setFont(selectDataSourceLabel.getFont().deriveFont(selectDataSourceLabel.getFont().getSize()-1f)); selectDataSourceLabel.setText(org.openide.util.NbBundle.getMessage(TermsExtractionDialog.class, "TermsExtractionDialog.selectDataSourceLabel.text")); // NOI18N + statusLabel.setFont(statusLabel.getFont().deriveFont(statusLabel.getFont().getSize()-1f)); + statusLabel.setText(org.openide.util.NbBundle.getMessage(TermsExtractionDialog.class, "TermsExtractionDialog.statusLabel.text")); // NOI18N + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); getContentPane().setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(extractTermsButton) + .addGroup(layout.createSequentialGroup() + .addComponent(extractTermsButton) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(statusLabel)) .addComponent(selectDataSourceLabel)) .addGap(0, 0, Short.MAX_VALUE)) .addGroup(layout.createSequentialGroup() - .addComponent(jScrollPane1) + .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 358, Short.MAX_VALUE) .addContainerGap()) ); layout.setVerticalGroup( @@ -239,7 +243,9 @@ class TermsExtractionDialog extends javax.swing.JDialog { .addGap(18, 18, 18) .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 65, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(extractTermsButton) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(extractTermsButton) + .addComponent(statusLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE)) .addContainerGap(20, Short.MAX_VALUE)) ); }// //GEN-END:initComponents @@ -252,6 +258,7 @@ class TermsExtractionDialog extends javax.swing.JDialog { private javax.swing.JButton extractTermsButton; private javax.swing.JScrollPane jScrollPane1; private javax.swing.JLabel selectDataSourceLabel; + private javax.swing.JLabel statusLabel; // End of variables declaration//GEN-END:variables private void searchAction(ActionEvent e) { @@ -270,6 +277,15 @@ class TermsExtractionDialog extends javax.swing.JDialog { * * @param saveResults Flag whether to save search results as KWS artifacts. */ + @NbBundle.Messages({ + "TermsExtractionDialog.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes.", + "TermsExtractionDialog.search.noFilesInIdxMsg2=No files are in index yet.
Try again later", + "TermsExtractionDialog.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress", + "TermsExtractionDialog.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway?", + "TermsExtractionDialog.startExport=Starting Unique Word Export", + "TermsExtractionDialog.export.error=Error During Unique Word Export", + "TermsExtractionDialog.exportComplete=Unique Word Export Complete" + }) void search() { boolean isIngestRunning = IngestManager.getInstance().isIngestRunning(); @@ -280,34 +296,34 @@ class TermsExtractionDialog extends javax.swing.JDialog { } if (filesIndexed == 0) { - // ELTODO if (isIngestRunning) { - KeywordSearchUtil.displayDialog(keywordSearchErrorDialogHeader, NbBundle.getMessage(this.getClass(), - "AbstractKeywordSearchPerformer.search.noFilesInIdxMsg", - KeywordSearchSettings.getUpdateFrequency().getTime()), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); + KeywordSearchUtil.displayDialog(errorDialogHeader, + Bundle.TermsExtractionDialog_search_noFilesInIdxMsg(KeywordSearchSettings.getUpdateFrequency().getTime()), + KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); } else { - KeywordSearchUtil.displayDialog(keywordSearchErrorDialogHeader, NbBundle.getMessage(this.getClass(), - "AbstractKeywordSearchPerformer.search.noFilesIdxdMsg"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); + KeywordSearchUtil.displayDialog(errorDialogHeader, Bundle.TermsExtractionDialog_search_noFilesInIdxMsg2(), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); } return; } //check if keyword search module ingest is running (indexing, etc) if (isIngestRunning) { - if (KeywordSearchUtil.displayConfirmDialog(org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.searchIngestInProgressTitle"), - NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.ingestInProgressBody"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN) == false) { + if (KeywordSearchUtil.displayConfirmDialog(Bundle.TermsExtractionDialog_search_searchIngestInProgressTitle(), + Bundle.TermsExtractionDialog_search_ingestInProgressBody(), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN) == false) { return; } } final Server server = KeywordSearch.getServer(); - Long dsID = Long.valueOf(4); Set selectedDs = getDataSourcesSelected(); try { + statusLabel.setText(Bundle.TermsExtractionDialog_startExport()); server.extractAllTermsForDataSource(selectedDs.iterator().next()); //ELTODO check if not empty } catch (Exception ex) { + statusLabel.setText(Bundle.TermsExtractionDialog_export_error()); Exceptions.printStackTrace(ex); } + statusLabel.setText(Bundle.TermsExtractionDialog_exportComplete()); } void addSearchButtonActionListener(ActionListener al) { @@ -334,7 +350,7 @@ class TermsExtractionDialog extends javax.swing.JDialog { /** * Update the dataSourceListModel */ - @NbBundle.Messages({"TermsExtractionPanel.selected=Ad Hoc Search data source filter is selected"}) + @NbBundle.Messages({"TermsExtractionPanel.selected=Data source filter is selected"}) void updateDataSourceListModel() throws NoCurrentCaseException, TskCoreException { dataSources = getDataSourceList(); getDataSourceListModel().removeAllElements(); From 04ca6411d9787dcbfc32754fef5fac909fd54b20 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Tue, 16 Feb 2021 14:33:39 -0500 Subject: [PATCH 07/37] More work --- .../keywordsearch/Bundle.properties-MERGED | 22 ++++++++- .../autopsy/keywordsearch/Server.java | 48 ++++++++++--------- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index f4c0be0948..72d49889a5 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -12,6 +12,7 @@ DropdownSingleTermSearchPanel.warning.text=Boundary characters ^ and $ do not ma DropdownSingleTermSearchPanel.warning.title=Warning ExtractAllTermsAction.getName.text=Extract Unique Words ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait +# {0} - Content name ExtractedContentPanel.SetMarkup.progress.loading=Loading text for {0} GlobalEditListPanel.editKeyword.title=Edit Keyword GlobalEditListPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] @@ -22,12 +23,16 @@ IndexedText.warningMessage.noTextAvailable=No in KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsOCR=Enable Optical Character Recognition (OCR) (Requires Windows 64-bit) KeywordSearchGlobalSettingsPanel.Title=Global Keyword Search Settings KeywordSearchIngestModule.init.badInitMsg=Keyword search server was not properly initialized, cannot run keyword search ingest. +# {0} - Reason for not connecting to Solr KeywordSearchIngestModule.init.exception.errConnToSolr.msg=Error connecting to SOLR server: {0}. +# {0} - Reason for not starting Solr KeywordSearchIngestModule.init.tryStopSolrMsg={0}
Please try stopping Java Solr processes if any exist and restart the application. KeywordSearchIngestModule.metadataTitle=METADATA KeywordSearchIngestModule.noOpenCase.errMsg=No open case available. KeywordSearchIngestModule.startUp.noOpenCore.msg=The index could not be opened or does not exist. +# {0} - schema version number KeywordSearchIngestModule.startupException.indexSchemaNotSupported=Adding text no longer supported for schema version {0} of the text index. +# {0} - Solr version number KeywordSearchIngestModule.startupException.indexSolrVersionNotSupported=Adding text no longer supported for Solr version {0} of the text index. KeywordSearchIngestModule.startupMessage.failedToGetIndexSchema=Failed to get schema version for text index. KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found. @@ -210,6 +215,7 @@ KeywordSearchSettings.propertiesNSRL.text={0}_NSRL KeywordSearchSettings.propertiesScripts.text={0}_Scripts NoOpenCoreException.err.noOpenSorlCore.msg=No currently open Solr core. SearchRunner.query.exception.msg=Error performing query: +# {0} - colelction name Server.deleteCore.exception.msg=Failed to delete Solr colelction {0} Server.exceptionMessage.unableToBackupCollection=Unable to backup Solr collection Server.exceptionMessage.unableToCreateCollection=Unable to create Solr collection @@ -336,15 +342,20 @@ SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr an SolrSearch.complete.msg=Text index successfully opened SolrSearch.creatingNewIndex.msg=Creating new text index SolrSearch.findingIndexes.msg=Looking for existing text index directories +# {0} - futureVersion +# {1} - currentVersion SolrSearch.futureIndexVersion.msg=The text index for the case is for Solr {0}. This version of Autopsy is compatible with Solr {1}. SolrSearch.indentifyingIndex.msg=Identifying text index to use SolrSearch.lookingForMetadata.msg=Looking for text index metadata file SolrSearch.openCore.msg=Opening text index. For large cases this may take several minutes. SolrSearch.readingIndexes.msg=Reading text index metadata file SolrSearch.unableToFindIndex.msg=Unable to find index that can be used for this case +# {0} - index folder path SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0} SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case +# {0} - case directory SolrSearchService.exceptionMessage.noIndexMetadata=Unable to create IndexMetaData from case directory: {0} +# {0} - collection name SolrSearchService.exceptionMessage.unableToDeleteCollection=Unable to delete collection {0} SolrSearchService.indexingError=Unable to index blackboard artifact. SolrSearchService.ServiceName=Solr Keyword Search Service @@ -378,9 +389,18 @@ ExtractedContentPanel.pageOfLabel.text=of ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) +TermsExtractionDialog.dialogErrorHeader=Error Extracting Unique Words +TermsExtractionDialog.export.error=Error During Unique Word Export +TermsExtractionDialog.exportComplete=Unique Word Export Complete +TermsExtractionDialog.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway? +TermsExtractionDialog.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes. +TermsExtractionDialog.search.noFilesInIdxMsg2=No files are in index yet.
Try again later +TermsExtractionDialog.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress TermsExtractionDialog.selectDataSourceLabel.text=Select Data Source: +TermsExtractionDialog.startExport=Starting Unique Word Export TermsExtractionDialog.statusLabel.text= -TermsExtractionPanel.selected=Ad Hoc Search data source filter is selected +TermsExtractionDialog.unableToGetDataSources.error=Unable to populate list of data sources +TermsExtractionPanel.selected=Data source filter is selected TextZoomPanel.zoomInButton.text= TextZoomPanel.zoomOutButton.text= TextZoomPanel.zoomResetButton.text=Reset diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 5135b3cf30..701d783e4b 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -2209,7 +2209,7 @@ public class Server { private void extractAllTermsForDataSource(Long dsObjId) throws IOException, SolrServerException, NoCurrentCaseException, KeywordSearchModuleException { String dataSourceId = Long.toString(dsObjId); - int numTerms = 400000; + /* ELTODO int numTerms = 400000; int termStep = 1000; SolrQuery query = new SolrQuery(); @@ -2220,7 +2220,7 @@ public class Server { query.addTermsField("text"); query.setTermsMinCount(0); - // ELTODO query.addFilterQuery("image_id:" + dataSourceId); + query.addFilterQuery("image_id:" + dataSourceId); QueryRequest request = new QueryRequest(query); TermsResponse response = request.process(queryClient).getTermsResponse(); @@ -2244,46 +2244,48 @@ public class Server { List listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList()); OpenOption[] options = new OpenOption[] { java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND }; - Files.write(serverFile, listTerms, options); - /*for (Term term : terms) { - try { - Files.write(serverFile, term.getTerm().getBytes(), options); - Files.write(serverFile, "\n".getBytes(), options); - } catch (IOException ex) { - throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex); //NON-NLS - } - }*/ + Files.write(serverFile, listTerms, options);*/ - // repeat the same thing but stepping through the terms - String firstTerm = ""; - Path serverFileIterated = Paths.get(caseDirectoryPath.toString(), "terms_"+numTerms+"_iterated.txt"); //NON-NLS + // step through the terms + Case currentCase = Case.getCurrentCaseThrows(); + File caseDirectoryPath = new File(currentCase.getOutputDirectory()); + Path serverFileIterated = Paths.get(caseDirectoryPath.toString(), "Unique Words from Data Source " + dataSourceId + ".txt"); //NON-NLS Files.deleteIfExists(serverFileIterated); - for (int step = 0; step < numTerms; step += termStep) { - query = new SolrQuery(); + OpenOption[] options = new OpenOption[] { java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND }; + + int termStep = 1000; + String firstTerm = ""; + while (true) { + SolrQuery query = new SolrQuery(); query.setRequestHandler("/terms"); query.setTerms(true); query.setTermsLimit(termStep); query.setTermsLower(firstTerm); query.setTermsLowerInclusive(false); + + // returned terms sorted by "index" order, which is the fastest way. Per Solr documentation: + // " Retrieving terms in index order is very fast since the implementation directly uses Lucene’s TermEnum to iterate over the term dictionary." + // All other sort criteria return very inconsistent and overlapping resuts. query.setTermsSortString("index"); - query.addTermsField("text"); + + // "text" field is the schema field that we populate with (lowercased) terms + query.addTermsField(Server.Schema.TEXT.toString()); query.setTermsMinCount(0); - // ELTODO query.addFilterQuery("image_id:" + dataSourceId); + query.addFilterQuery(Server.Schema.IMAGE_ID.toString() + ":" + dataSourceId); - request = new QueryRequest(query); - response = request.process(queryClient).getTermsResponse(); - terms = response.getTerms("text"); + QueryRequest request = new QueryRequest(query); + TermsResponse response = request.process(queryClient).getTermsResponse(); + List terms = response.getTerms(Server.Schema.TEXT.toString()); if (terms == null || terms.isEmpty()) { - logger.log(Level.WARNING, "No unique terms/words returned for data source ID: " + dataSourceId); //NON-NLS break; } // set the first term for the next query firstTerm = terms.get(terms.size()-1).getTerm(); - listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList()); + List listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList()); Files.write(serverFileIterated, listTerms, options); } } From 571a4c314e9e88e945a6547412eb5a5e6d964691 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 19 Feb 2021 18:35:21 -0500 Subject: [PATCH 08/37] Using report infrastructure instead --- .../keywordsearch/Bundle.properties-MERGED | 12 +- .../keywordsearch/ExtractAllTermsAction.java | 67 --- .../keywordsearch/ExtractAllTermsReport.java | 129 ++++++ .../autopsy/keywordsearch/Server.java | 75 +--- .../keywordsearch/TermsExtractionDialog.form | 120 ------ .../keywordsearch/TermsExtractionDialog.java | 393 ------------------ .../sleuthkit/autopsy/keywordsearch/layer.xml | 4 + 7 files changed, 165 insertions(+), 635 deletions(-) delete mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java create mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java delete mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form delete mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 72d49889a5..e130c9052a 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -5,12 +5,20 @@ AccountsText.nextPage.exception.msg=No next page. AccountsText.previousItem.exception.msg=No previous item. AccountsText.previousPage.exception.msg=No previous page. CannotRunFileTypeDetection=Unable to run file type detection. -CTL_ExtractAllTermsAction=Extract Unique Words DropdownListSearchPanel.selected=Ad Hoc Search data source filter is selected DropdownSingleTermSearchPanel.selected=Ad Hoc Search data source filter is selected DropdownSingleTermSearchPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] DropdownSingleTermSearchPanel.warning.title=Warning -ExtractAllTermsAction.getName.text=Extract Unique Words +ExtractAllTermsReport.error.unableToOpenCase=Exception while getting open case. +ExtractAllTermsReport.export.error=Error During Unique Word Export +ExtractAllTermsReport.exportComplete=Unique Word Export Complete +ExtractAllTermsReport.getName.text=Extract Unique Words +ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms... +ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway? +ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes. +ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet.
Try again later +ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress +ExtractAllTermsReport.startExport=Starting Unique Word Export ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait # {0} - Content name ExtractedContentPanel.SetMarkup.progress.loading=Loading text for {0} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java deleted file mode 100755 index c68061f8d5..0000000000 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsAction.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2021 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.keywordsearch; - -import java.awt.event.ActionEvent; -import org.openide.awt.ActionID; -import org.openide.awt.ActionReference; -import org.openide.awt.ActionRegistration; -import org.openide.util.HelpCtx; -import org.openide.util.NbBundle; -import org.openide.util.actions.CallableSystemAction; -import org.sleuthkit.autopsy.casemodule.Case; - -/** - * Action for accessing the Search Other Cases dialog. - */ -@ActionID(category = "Tools", id = "org.sleuthkit.autopsy.keywordsearch.ExtractAllTermsAction") -@ActionRegistration(displayName = "#CTL_OtherExtractAllTermsAction=Extract Unique Words", lazy = false) -@ActionReference(path = "Menu/Tools", position = 202) -@NbBundle.Messages({"CTL_ExtractAllTermsAction=Extract Unique Words"}) -public class ExtractAllTermsAction extends CallableSystemAction { - - @Override - public boolean isEnabled() { - return Case.isCaseOpen(); - } - - @Override - public void actionPerformed(ActionEvent event) { - performAction(); - } - - @Override - public void performAction() { - TermsExtractionDialog listsPanel = TermsExtractionDialog.getDefault(); - listsPanel.display(); - } - - @NbBundle.Messages({ - "ExtractAllTermsAction.getName.text=Extract Unique Words"}) - @Override - public String getName() { - return Bundle.ExtractAllTermsAction_getName_text(); - } - - @Override - public HelpCtx getHelpCtx() { - return HelpCtx.DEFAULT_HELP; - } - -} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java new file mode 100755 index 0000000000..9debf161ac --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java @@ -0,0 +1,129 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.util.logging.Level; +import org.openide.util.Exceptions; +import org.openide.util.NbBundle; +import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.IngestManager; +import org.sleuthkit.autopsy.report.GeneralReportModule; +import org.sleuthkit.autopsy.report.GeneralReportSettings; +import org.sleuthkit.autopsy.report.ReportProgressPanel; + +/** + * Instances of this class plug in to the reporting infrastructure to provide a + * convenient way to extract all unique terms from Solr index. + */ +@ServiceProvider(service = GeneralReportModule.class) +public class ExtractAllTermsReport implements GeneralReportModule { + + private static final Logger logger = Logger.getLogger(ExtractAllTermsReport.class.getName()); + + @NbBundle.Messages({ + "ExtractAllTermsReport.getName.text=Extract Unique Words"}) + @Override + public String getName() { + return Bundle.ExtractAllTermsReport_getName_text(); + } + + @NbBundle.Messages({ + "ExtractAllTermsReport.error.unableToOpenCase=Exception while getting open case.", + "ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes.", + "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet.
Try again later", + "ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress", + "ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway?", + "ExtractAllTermsReport.startExport=Starting Unique Word Export", + "ExtractAllTermsReport.export.error=Error During Unique Word Export", + "ExtractAllTermsReport.exportComplete=Unique Word Export Complete" + }) + @Override + public void generateReport(GeneralReportSettings settings, ReportProgressPanel progressPanel) { + Case openCase; + try { + openCase = Case.getCurrentCaseThrows(); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_error_unableToOpenCase()); + return; + } + progressPanel.setIndeterminate(true); + progressPanel.start(); + progressPanel.updateStatusLabel("Extracting unique words..."); + + boolean isIngestRunning = IngestManager.getInstance().isIngestRunning(); + + int filesIndexed = 0; + try { // see if another node added any indexed files + filesIndexed = KeywordSearch.getServer().queryNumIndexedFiles(); + } catch (KeywordSearchModuleException | NoOpenCoreException ignored) { + } + + if (filesIndexed == 0) { + if (isIngestRunning) { + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_search_noFilesInIdxMsg(KeywordSearchSettings.getUpdateFrequency().getTime())); + } else { + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_search_noFilesInIdxMsg2()); + } + progressPanel.setIndeterminate(false); + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR); + return; + } + + //check if keyword search module ingest is running (indexing, etc) + if (isIngestRunning) { + if (KeywordSearchUtil.displayConfirmDialog(Bundle.ExtractAllTermsReport_search_searchIngestInProgressTitle(), + Bundle.ExtractAllTermsReport_search_ingestInProgressBody(), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN) == false) { + return; + } + } + + final Server server = KeywordSearch.getServer(); + try { + progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_startExport()); + server.extractAllTermsForDataSource(settings, progressPanel); + } catch (Exception ex) { + progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_export_error()); + Exceptions.printStackTrace(ex); // ELTODO + } + progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_exportComplete()); + + progressPanel.setIndeterminate(false); + progressPanel.complete(ReportProgressPanel.ReportStatus.COMPLETE); + } + + @Override + public boolean supportsDataSourceSelection() { + return false; + } + + @Override + public String getDescription() { + return "Extracts all unique words out of the current case. NOTE: The extracted words are lower-cased."; + } + + @Override + public String getRelativeFilePath() { + return "Unique Words.txt"; // ELTODO + } + +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 701d783e4b..0f9a59a881 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -43,10 +43,8 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Random; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -61,7 +59,6 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; @@ -98,6 +95,8 @@ import org.sleuthkit.autopsy.coreutils.ThreadUtils; import org.sleuthkit.autopsy.healthmonitor.HealthMonitor; import org.sleuthkit.autopsy.healthmonitor.TimingMetric; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException; +import org.sleuthkit.autopsy.report.GeneralReportSettings; +import org.sleuthkit.autopsy.report.ReportProgressPanel; import org.sleuthkit.datamodel.Content; /** @@ -1843,17 +1842,18 @@ public class Server { /** * Extract all unique terms/words for a given data source. * - * @param dataSourceId to process + * @param settings GeneralReportSettings to use + * @param progressPanel ReportProgressPanel to update * * @throws NoOpenCoreException */ - void extractAllTermsForDataSource(Long dataSourceId) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException, NoCurrentCaseException { + void extractAllTermsForDataSource(GeneralReportSettings settings, ReportProgressPanel progressPanel) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException, NoCurrentCaseException { try { currentCoreLock.writeLock().lock(); if (null == currentCollection) { throw new NoOpenCoreException(); } - currentCollection.extractAllTermsForDataSource(dataSourceId); + currentCollection.extractAllTermsForDataSource(settings, progressPanel); } finally { currentCoreLock.writeLock().unlock(); } @@ -2206,54 +2206,18 @@ public class Server { queryClient.deleteByQuery(deleteQuery); } - private void extractAllTermsForDataSource(Long dsObjId) throws IOException, SolrServerException, NoCurrentCaseException, KeywordSearchModuleException { - String dataSourceId = Long.toString(dsObjId); + @NbBundle.Messages({ + "ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms..." + }) + private void extractAllTermsForDataSource(GeneralReportSettings settings, ReportProgressPanel progressPanel) throws IOException, SolrServerException, NoCurrentCaseException, KeywordSearchModuleException { - /* ELTODO int numTerms = 400000; - int termStep = 1000; - - SolrQuery query = new SolrQuery(); - query.setRequestHandler("/terms"); - query.setTerms(true); - query.setTermsLimit(numTerms); - query.setTermsSortString("index"); - query.addTermsField("text"); - query.setTermsMinCount(0); - - query.addFilterQuery("image_id:" + dataSourceId); - - QueryRequest request = new QueryRequest(query); - TermsResponse response = request.process(queryClient).getTermsResponse(); - List terms = response.getTerms("text"); - - if (terms == null || terms.isEmpty()) { - logger.log(Level.WARNING, "No unique terms/words returned for data source ID: " + dataSourceId); //NON-NLS - return; - } - - //Term term = terms.get(0); - //String word = term.getTerm(); - //long frequency = term.getFrequency(); - - // Write the server data to a file - Case currentCase = Case.getCurrentCaseThrows(); - File caseDirectoryPath = new File(currentCase.getOutputDirectory()); - Path serverFile = Paths.get(caseDirectoryPath.toString(), "terms_"+numTerms+"_whole.txt"); //NON-NLS - Files.deleteIfExists(serverFile); - - List listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList()); - - OpenOption[] options = new OpenOption[] { java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND }; - Files.write(serverFile, listTerms, options);*/ - - // step through the terms - Case currentCase = Case.getCurrentCaseThrows(); - File caseDirectoryPath = new File(currentCase.getOutputDirectory()); - Path serverFileIterated = Paths.get(caseDirectoryPath.toString(), "Unique Words from Data Source " + dataSourceId + ".txt"); //NON-NLS + // step through the terms + Path serverFileIterated = Paths.get(settings.getReportDirectoryPath(), "Unique Words.txt"); //NON-NLS Files.deleteIfExists(serverFileIterated); OpenOption[] options = new OpenOption[] { java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND }; int termStep = 1000; + long numExtractedTerms = 0; String firstTerm = ""; while (true) { SolrQuery query = new SolrQuery(); @@ -2263,8 +2227,8 @@ public class Server { query.setTermsLower(firstTerm); query.setTermsLowerInclusive(false); - // returned terms sorted by "index" order, which is the fastest way. Per Solr documentation: - // " Retrieving terms in index order is very fast since the implementation directly uses Lucene’s TermEnum to iterate over the term dictionary." + // Returned terms sorted by "index" order, which is the fastest way. Per Solr documentation: + // "Retrieving terms in index order is very fast since the implementation directly uses Lucene’s TermEnum to iterate over the term dictionary." // All other sort criteria return very inconsistent and overlapping resuts. query.setTermsSortString("index"); @@ -2272,13 +2236,16 @@ public class Server { query.addTermsField(Server.Schema.TEXT.toString()); query.setTermsMinCount(0); - query.addFilterQuery(Server.Schema.IMAGE_ID.toString() + ":" + dataSourceId); + // Unfortunatelly Solr "terms queries" do not support any filtering so we can't filter by data source this way. + // query.addFilterQuery(Server.Schema.IMAGE_ID.toString() + ":" + dataSourceId); QueryRequest request = new QueryRequest(query); TermsResponse response = request.process(queryClient).getTermsResponse(); List terms = response.getTerms(Server.Schema.TEXT.toString()); if (terms == null || terms.isEmpty()) { + numExtractedTerms += terms.size(); + progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_numberExtractedTerms(numExtractedTerms)); break; } @@ -2287,6 +2254,9 @@ public class Server { List listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList()); Files.write(serverFileIterated, listTerms, options); + + numExtractedTerms += termStep; + progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_numberExtractedTerms(numExtractedTerms)); } } @@ -2325,7 +2295,6 @@ public class Server { * * @throws KeywordSearchModuleException */ - // ELTODO DECIDE ON SYNCHRONIZATION private void sendBufferedDocs(List docBuffer) throws KeywordSearchModuleException { if (docBuffer.isEmpty()) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form deleted file mode 100755 index c398b11e0a..0000000000 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.form +++ /dev/null @@ -1,120 +0,0 @@ - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java deleted file mode 100755 index 52908682cc..0000000000 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsExtractionDialog.java +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2021 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.keywordsearch; - -import java.awt.Cursor; -import java.awt.EventQueue; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.beans.PropertyChangeEvent; -import java.beans.PropertyChangeListener; -import java.io.File; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Level; -import javax.swing.DefaultListModel; -import javax.swing.ListSelectionModel; -import javax.swing.event.ListSelectionEvent; -import org.openide.util.Exceptions; -import org.openide.util.NbBundle; -import org.openide.windows.WindowManager; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.ingest.IngestManager; -import org.sleuthkit.datamodel.DataSource; -import org.sleuthkit.datamodel.TskCoreException; - -/** - * Viewer panel widget for keyword lists that is used in the ingest config and - * options area. - */ -@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives -@NbBundle.Messages({ - "TermsExtractionDialog.dialogErrorHeader=Error Extracting Unique Words"}) -class TermsExtractionDialog extends javax.swing.JDialog { - - private static final Logger logger = Logger.getLogger(TermsExtractionDialog.class.getName()); - - // ELTODO - private final String errorDialogHeader = Bundle.TermsExtractionDialog_dialogErrorHeader(); - private static TermsExtractionDialog instance; - private ActionListener searchAddListener; - private boolean ingestRunning; - private final Map dataSourceMap = new HashMap<>(); - private List dataSources = new ArrayList<>(); - private final DefaultListModel dataSourceListModel = new DefaultListModel<>(); - - /** - * Creates new form TermsExtractionPanel - */ - private TermsExtractionDialog() { - initComponents(); - customizeComponents(); - dataSourceList.setModel(getDataSourceListModel()); - - dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { - firePropertyChange(Bundle.TermsExtractionPanel_selected(), null, null); // ELTODO - }); - } - - static synchronized TermsExtractionDialog getDefault() { - if (instance == null) { - instance = new TermsExtractionDialog(); - } - return instance; - } - - /** - * Display the Search Other Cases dialog. - */ - void display() { - updateComponents(); - this.setLocationRelativeTo(WindowManager.getDefault().getMainWindow()); - setVisible(true); - } - - private void customizeComponents() { - - ingestRunning = IngestManager.getInstance().isIngestRunning(); - updateComponents(); - - IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { - @Override - public void propertyChange(PropertyChangeEvent evt) { - Object source = evt.getSource(); - if (source instanceof String && ((String) source).equals("LOCAL")) { //NON-NLS - EventQueue.invokeLater(() -> { - ingestRunning = IngestManager.getInstance().isIngestRunning(); - updateComponents(); - }); - } - } - }); - - searchAddListener = new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - if (ingestRunning) { -// ELTODO IngestSearchRunner.getInstance().addKeywordListsToAllJobs(listsTableModel.getSelectedLists()); - logger.log(Level.INFO, "Ingest is running"); //NON-NLS - } else { - searchAction(e); - } - } - }; - - extractTermsButton.addActionListener(searchAddListener); - } - - @NbBundle.Messages({ - "TermsExtractionDialog.unableToGetDataSources.error=Unable to populate list of data sources"}) - private void updateComponents() { - statusLabel.setText(""); - ingestRunning = IngestManager.getInstance().isIngestRunning(); - - try { - updateDataSourceListModel(); - } catch (NoCurrentCaseException | TskCoreException ex) { - statusLabel.setText(Bundle.TermsExtractionDialog_unableToGetDataSources_error()); - logger.log(Level.SEVERE, "Unable to populate list of data sources", ex); //NON-NLS - } - } - - /** - * Get a list of data source display names. - * - * @return The list of data source name - */ - synchronized List getDataSourceArray() { - List dsList = new ArrayList<>(); - Collections.sort(this.dataSources, (DataSource ds1, DataSource ds2) -> ds1.getName().compareTo(ds2.getName())); - for (DataSource ds : dataSources) { - String dsName = ds.getName(); - File dataSourceFullName = new File(dsName); - String displayName = dataSourceFullName.getName(); - dataSourceMap.put(ds.getId(), displayName); - dsList.add(displayName); - } - return dsList; - } - - /** - * Set dataSources - * @param dataSources A list of DataSource - */ - synchronized void setDataSources(List dataSources) { - this.dataSources = dataSources; - } - - /** - * Get dataSourceMap with object id and data source display name. - * - * @return The list of data source name - */ - Map getDataSourceMap() { - return dataSourceMap; - } - - /** - * Get a list of DataSourceListModel - * @return A list of DataSourceListModel - */ - final DefaultListModel getDataSourceListModel() { - return dataSourceListModel; - } - - /** - * This method is called from within the constructor to initialize the form. - * WARNING: Do NOT modify this code. The content of this method is always - * regenerated by the Form Editor. - */ - @SuppressWarnings("unchecked") - // //GEN-BEGIN:initComponents - private void initComponents() { - - extractTermsButton = new javax.swing.JButton(); - jScrollPane1 = new javax.swing.JScrollPane(); - dataSourceList = new javax.swing.JList<>(); - selectDataSourceLabel = new javax.swing.JLabel(); - statusLabel = new javax.swing.JLabel(); - - setSize(new java.awt.Dimension(500, 200)); - - extractTermsButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/keywordsearch/search-icon.png"))); // NOI18N - extractTermsButton.setText(org.openide.util.NbBundle.getMessage(TermsExtractionDialog.class, "KeywordSearchListsViewerPanel.searchAddButton.text")); // NOI18N - extractTermsButton.addActionListener(new java.awt.event.ActionListener() { - public void actionPerformed(java.awt.event.ActionEvent evt) { - extractTermsButtonActionPerformed(evt); - } - }); - - dataSourceList.setMinimumSize(new java.awt.Dimension(0, 200)); - jScrollPane1.setViewportView(dataSourceList); - - selectDataSourceLabel.setFont(selectDataSourceLabel.getFont().deriveFont(selectDataSourceLabel.getFont().getSize()-1f)); - selectDataSourceLabel.setText(org.openide.util.NbBundle.getMessage(TermsExtractionDialog.class, "TermsExtractionDialog.selectDataSourceLabel.text")); // NOI18N - - statusLabel.setFont(statusLabel.getFont().deriveFont(statusLabel.getFont().getSize()-1f)); - statusLabel.setText(org.openide.util.NbBundle.getMessage(TermsExtractionDialog.class, "TermsExtractionDialog.statusLabel.text")); // NOI18N - - javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); - getContentPane().setLayout(layout); - layout.setHorizontalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addComponent(extractTermsButton) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(statusLabel)) - .addComponent(selectDataSourceLabel)) - .addGap(0, 0, Short.MAX_VALUE)) - .addGroup(layout.createSequentialGroup() - .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 358, Short.MAX_VALUE) - .addContainerGap()) - ); - layout.setVerticalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addGap(15, 15, 15) - .addComponent(selectDataSourceLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGap(18, 18, 18) - .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 65, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(extractTermsButton) - .addComponent(statusLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addContainerGap(20, Short.MAX_VALUE)) - ); - }// //GEN-END:initComponents - - private void extractTermsButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_extractTermsButtonActionPerformed - }//GEN-LAST:event_extractTermsButtonActionPerformed - - // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JList dataSourceList; - private javax.swing.JButton extractTermsButton; - private javax.swing.JScrollPane jScrollPane1; - private javax.swing.JLabel selectDataSourceLabel; - private javax.swing.JLabel statusLabel; - // End of variables declaration//GEN-END:variables - - private void searchAction(ActionEvent e) { - setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - - try { - search(); - } finally { - setCursor(null); - } - } - - /** - * Performs the search using the selected keywords. Creates a - * DataResultTopComponent with the results. - * - * @param saveResults Flag whether to save search results as KWS artifacts. - */ - @NbBundle.Messages({ - "TermsExtractionDialog.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes.", - "TermsExtractionDialog.search.noFilesInIdxMsg2=No files are in index yet.
Try again later", - "TermsExtractionDialog.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress", - "TermsExtractionDialog.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway?", - "TermsExtractionDialog.startExport=Starting Unique Word Export", - "TermsExtractionDialog.export.error=Error During Unique Word Export", - "TermsExtractionDialog.exportComplete=Unique Word Export Complete" - }) - void search() { - boolean isIngestRunning = IngestManager.getInstance().isIngestRunning(); - - int filesIndexed = 0; - try { // see if another node added any indexed files - filesIndexed = KeywordSearch.getServer().queryNumIndexedFiles(); - } catch (KeywordSearchModuleException | NoOpenCoreException ignored) { - } - - if (filesIndexed == 0) { - if (isIngestRunning) { - KeywordSearchUtil.displayDialog(errorDialogHeader, - Bundle.TermsExtractionDialog_search_noFilesInIdxMsg(KeywordSearchSettings.getUpdateFrequency().getTime()), - KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); - } else { - KeywordSearchUtil.displayDialog(errorDialogHeader, Bundle.TermsExtractionDialog_search_noFilesInIdxMsg2(), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); - } - return; - } - - //check if keyword search module ingest is running (indexing, etc) - if (isIngestRunning) { - if (KeywordSearchUtil.displayConfirmDialog(Bundle.TermsExtractionDialog_search_searchIngestInProgressTitle(), - Bundle.TermsExtractionDialog_search_ingestInProgressBody(), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN) == false) { - return; - } - } - - final Server server = KeywordSearch.getServer(); - Set selectedDs = getDataSourcesSelected(); - try { - statusLabel.setText(Bundle.TermsExtractionDialog_startExport()); - server.extractAllTermsForDataSource(selectedDs.iterator().next()); //ELTODO check if not empty - } catch (Exception ex) { - statusLabel.setText(Bundle.TermsExtractionDialog_export_error()); - Exceptions.printStackTrace(ex); - } - statusLabel.setText(Bundle.TermsExtractionDialog_exportComplete()); - } - - void addSearchButtonActionListener(ActionListener al) { - extractTermsButton.addActionListener(al); - } - - /** - * Get a set of data source object ids that are selected. - * @return A set of selected object ids. - */ - Set getDataSourcesSelected() { - Set dataSourceObjIdSet = new HashSet<>(); - for (Long key : getDataSourceMap().keySet()) { - String value = getDataSourceMap().get(key); - for (String dataSource : this.dataSourceList.getSelectedValuesList()) { - if (value.equals(dataSource)) { - dataSourceObjIdSet.add(key); - } - } - } - return dataSourceObjIdSet; - } - - /** - * Update the dataSourceListModel - */ - @NbBundle.Messages({"TermsExtractionPanel.selected=Data source filter is selected"}) - void updateDataSourceListModel() throws NoCurrentCaseException, TskCoreException { - dataSources = getDataSourceList(); - getDataSourceListModel().removeAllElements(); - for (String dsName : getDataSourceArray()) { - getDataSourceListModel().addElement(dsName); - } - setComponentsEnabled(); - firePropertyChange(Bundle.TermsExtractionPanel_selected(), null, null); // ELTODO - } - - /** - * Get a list of DataSource from case database - * @return A list of DataSource - * @throws NoCurrentCaseException - * @throws TskCoreException - */ - private synchronized List getDataSourceList() throws NoCurrentCaseException, TskCoreException { - Case openCase = Case.getCurrentCaseThrows(); - return openCase.getSleuthkitCase().getDataSources(); - } - - /** - * Set the dataSourceList enabled if there are data sources to select - */ - private void setComponentsEnabled() { - - this.dataSourceList.setEnabled(false); - extractTermsButton.setEnabled(false); - if (getDataSourceListModel().size() > 1) { - this.dataSourceList.setEnabled(true); - this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); - dataSourceList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION); // do not allow multiselect - extractTermsButton.setEnabled(true); - } else if (getDataSourceListModel().size() == 1) { - this.dataSourceList.setEnabled(false); - this.dataSourceList.setSelectionInterval(0, 0); - extractTermsButton.setEnabled(true); - } - } -} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/layer.xml b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/layer.xml index bde41b641f..8effaba623 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/layer.xml +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/layer.xml @@ -7,6 +7,10 @@ ====================================================== --> + + + + From cb95f8b093fe5ca49098f6b5253bfe9776884f26 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Tue, 23 Feb 2021 18:25:52 -0500 Subject: [PATCH 09/37] More work --- .../keywordsearch/Bundle.properties-MERGED | 3 +- .../keywordsearch/ExtractAllTermsReport.java | 34 +++++++++++-------- .../autopsy/keywordsearch/Server.java | 23 ++++++++----- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index e130c9052a..e3f51bbb12 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -9,6 +9,7 @@ DropdownListSearchPanel.selected=Ad Hoc Search data source filter is selected DropdownSingleTermSearchPanel.selected=Ad Hoc Search data source filter is selected DropdownSingleTermSearchPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] DropdownSingleTermSearchPanel.warning.title=Warning +ExtractAllTermsReport.error.noOpenCase=No currently open case. ExtractAllTermsReport.error.unableToOpenCase=Exception while getting open case. ExtractAllTermsReport.export.error=Error During Unique Word Export ExtractAllTermsReport.exportComplete=Unique Word Export Complete @@ -20,7 +21,6 @@ ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. < ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress ExtractAllTermsReport.startExport=Starting Unique Word Export ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait -# {0} - Content name ExtractedContentPanel.SetMarkup.progress.loading=Loading text for {0} GlobalEditListPanel.editKeyword.title=Edit Keyword GlobalEditListPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] @@ -228,6 +228,7 @@ Server.deleteCore.exception.msg=Failed to delete Solr colelction {0} Server.exceptionMessage.unableToBackupCollection=Unable to backup Solr collection Server.exceptionMessage.unableToCreateCollection=Unable to create Solr collection Server.exceptionMessage.unableToRestoreCollection=Unable to restore Solr collection +Server.getAllTerms.error=Extraction of all unique Solr terms failed: Server.start.exception.cantStartSolr.msg=Could not start Solr server process Server.start.exception.cantStartSolr.msg2=Could not start Solr server process Server.isRunning.exception.errCheckSolrRunning.msg=Error checking if Solr server is running diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java index 9debf161ac..b1aa714af7 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java @@ -18,12 +18,12 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.logging.Level; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.report.GeneralReportModule; @@ -38,6 +38,7 @@ import org.sleuthkit.autopsy.report.ReportProgressPanel; public class ExtractAllTermsReport implements GeneralReportModule { private static final Logger logger = Logger.getLogger(ExtractAllTermsReport.class.getName()); + private static final String OUTPUT_FILE_NAME = "Unique Words.txt"; @NbBundle.Messages({ "ExtractAllTermsReport.getName.text=Extract Unique Words"}) @@ -45,9 +46,9 @@ public class ExtractAllTermsReport implements GeneralReportModule { public String getName() { return Bundle.ExtractAllTermsReport_getName_text(); } - + @NbBundle.Messages({ - "ExtractAllTermsReport.error.unableToOpenCase=Exception while getting open case.", + "ExtractAllTermsReport.error.noOpenCase=No currently open case.", "ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes.", "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet.
Try again later", "ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress", @@ -58,14 +59,13 @@ public class ExtractAllTermsReport implements GeneralReportModule { }) @Override public void generateReport(GeneralReportSettings settings, ReportProgressPanel progressPanel) { - Case openCase; - try { - openCase = Case.getCurrentCaseThrows(); - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_error_unableToOpenCase()); + + if (!Case.isCaseOpen()) { + logger.log(Level.SEVERE, "No open case when attempting to run {0} report", Bundle.ExtractAllTermsReport_getName_text()); //NON-NLS + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_error_noOpenCase()); return; } + progressPanel.setIndeterminate(true); progressPanel.start(); progressPanel.updateStatusLabel("Extracting unique words..."); @@ -93,6 +93,8 @@ public class ExtractAllTermsReport implements GeneralReportModule { if (isIngestRunning) { if (KeywordSearchUtil.displayConfirmDialog(Bundle.ExtractAllTermsReport_search_searchIngestInProgressTitle(), Bundle.ExtractAllTermsReport_search_ingestInProgressBody(), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN) == false) { + progressPanel.setIndeterminate(false); + progressPanel.complete(ReportProgressPanel.ReportStatus.CANCELED); return; } } @@ -100,10 +102,14 @@ public class ExtractAllTermsReport implements GeneralReportModule { final Server server = KeywordSearch.getServer(); try { progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_startExport()); - server.extractAllTermsForDataSource(settings, progressPanel); - } catch (Exception ex) { + Path outputFile = Paths.get(settings.getReportDirectoryPath(), getRelativeFilePath()); + server.extractAllTermsForDataSource(outputFile, progressPanel); + } catch (KeywordSearchModuleException | NoOpenCoreException ex) { + logger.log(Level.SEVERE, "Exception while extracting all terms", ex); //NON-NLS progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_export_error()); - Exceptions.printStackTrace(ex); // ELTODO + progressPanel.setIndeterminate(false); + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR); + return; } progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_exportComplete()); @@ -123,7 +129,7 @@ public class ExtractAllTermsReport implements GeneralReportModule { @Override public String getRelativeFilePath() { - return "Unique Words.txt"; // ELTODO + return OUTPUT_FILE_NAME; } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 0dcecd584e..2772776ed8 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -1791,22 +1791,30 @@ public class Server { /** * Extract all unique terms/words for a given data source. * - * @param settings GeneralReportSettings to use + * @param outputFile Absolute path to where the store the output * @param progressPanel ReportProgressPanel to update * * @throws NoOpenCoreException */ - void extractAllTermsForDataSource(GeneralReportSettings settings, ReportProgressPanel progressPanel) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException, NoCurrentCaseException { + @NbBundle.Messages({ + "Server.getAllTerms.error=Extraction of all unique Solr terms failed:"}) + void extractAllTermsForDataSource(Path outputFile, ReportProgressPanel progressPanel) throws KeywordSearchModuleException, NoOpenCoreException { try { currentCoreLock.writeLock().lock(); if (null == currentCollection) { throw new NoOpenCoreException(); } - currentCollection.extractAllTermsForDataSource(settings, progressPanel); + try { + currentCollection.extractAllTermsForDataSource(outputFile, progressPanel); + } catch (Exception ex) { + // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions + logger.log(Level.SEVERE, "Extraction of all unique Solr terms failed: ", ex); //NON-NLS + throw new KeywordSearchModuleException(Bundle.Server_getAllTerms_error(), ex); + } } finally { currentCoreLock.writeLock().unlock(); } - } + } /** * Get the text contents of the given file as stored in SOLR. @@ -2158,11 +2166,10 @@ public class Server { @NbBundle.Messages({ "ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms..." }) - private void extractAllTermsForDataSource(GeneralReportSettings settings, ReportProgressPanel progressPanel) throws IOException, SolrServerException, NoCurrentCaseException, KeywordSearchModuleException { + private void extractAllTermsForDataSource(Path outputFile, ReportProgressPanel progressPanel) throws IOException, SolrServerException, NoCurrentCaseException, KeywordSearchModuleException { // step through the terms - Path serverFileIterated = Paths.get(settings.getReportDirectoryPath(), "Unique Words.txt"); //NON-NLS - Files.deleteIfExists(serverFileIterated); + Files.deleteIfExists(outputFile); OpenOption[] options = new OpenOption[] { java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND }; int termStep = 1000; @@ -2202,7 +2209,7 @@ public class Server { firstTerm = terms.get(terms.size()-1).getTerm(); List listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList()); - Files.write(serverFileIterated, listTerms, options); + Files.write(outputFile, listTerms, options); numExtractedTerms += termStep; progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_numberExtractedTerms(numExtractedTerms)); From 748d2b7ad376e5aee300eeaec24403fa5b365a60 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 23 Feb 2021 20:12:44 -0500 Subject: [PATCH 10/37] working view context action --- .../datamodel/ContentNodeSelectionInfo.java | 2 +- .../datamodel/DataSourcesByTypeNode.java | 2 +- .../directorytree/ViewContextAction.java | 64 +++++++++++++++---- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentNodeSelectionInfo.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentNodeSelectionInfo.java index e5b8718931..fe92235454 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentNodeSelectionInfo.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentNodeSelectionInfo.java @@ -50,7 +50,7 @@ public class ContentNodeSelectionInfo implements NodeSelectionInfo { @Override public boolean matches(Node candidateNode) { Content content = candidateNode.getLookup().lookup(Content.class); - return content.getId() == contentId; + return (content != null && content.getId() == contentId); } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java index 856b6eec96..3152573403 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java @@ -93,7 +93,7 @@ public class DataSourcesByTypeNode extends DisplayableItemNode { } - private static final String NAME = Bundle.DataSourcesHostsNode_name(); + public static final String NAME = Bundle.DataSourcesHostsNode_name(); /** * Main constructor. diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java index ef8adfa319..7a105f389c 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java @@ -22,10 +22,13 @@ import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.beans.PropertyVetoException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.logging.Level; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.AbstractAction; import org.openide.nodes.AbstractNode; @@ -43,6 +46,7 @@ import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode; import org.sleuthkit.autopsy.datamodel.AbstractFsContentNode; import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode; import org.sleuthkit.autopsy.datamodel.ContentNodeSelectionInfo; +import org.sleuthkit.autopsy.datamodel.DataSourcesByTypeNode; import org.sleuthkit.autopsy.datamodel.DataSourcesNode; import org.sleuthkit.autopsy.datamodel.DisplayableItemNode; import org.sleuthkit.autopsy.datamodel.RootContentChildren; @@ -52,6 +56,8 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentVisitor; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.FileSystem; +import org.sleuthkit.datamodel.Host; +import org.sleuthkit.datamodel.Person; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -186,17 +192,20 @@ public class ViewContextAction extends AbstractAction { DataSource datasource = skCase.getDataSource(contentDSObjid); dsname = datasource.getName(); Children rootChildren = treeViewExplorerMgr.getRootContext().getChildren(); - + if (null != parentContent) { // the tree view needs to be searched to find the parent treeview node. /* NOTE: we can't do a lookup by data source name here, becase if there are multiple data sources with the same name, then "getChildren().findChild(dsname)" simply returns the first one that it finds. Instead we have to loop over all data sources with that name, and make sure we find the correct one. - */ - for (int i = 0; i < rootChildren.getNodesCount(); i++) { + */ + List dataSourceLevelNodes = Stream.of(rootChildren.getNodes()) + .flatMap(rootNode -> getDataSourceLevelNodes(rootNode).stream()) + .collect(Collectors.toList()); + + for (Node treeNode : dataSourceLevelNodes) { // in the root, look for a data source node with the name of interest - Node treeNode = rootChildren.getNodeAt(i); if (!(treeNode.getName().equals(dsname))) { continue; } @@ -234,13 +243,19 @@ public class ViewContextAction extends AbstractAction { } } else { // Classic view // Start the search at the DataSourcesNode - parentTreeViewNode = treeViewExplorerMgr.getRootContext().getChildren().findChild(DataSourcesNode.NAME); - - if (null != parentContent) { - // the tree view needs to be searched to find the parent treeview node. - Node potentialParentTreeViewNode = findParentNodeInTree(parentContent, parentTreeViewNode); - if (potentialParentTreeViewNode != null) { - parentTreeViewNode = potentialParentTreeViewNode; + Children rootChildren = treeViewExplorerMgr.getRootContext().getChildren(); + Node rootDsNode = rootChildren == null ? null : rootChildren.findChild(DataSourcesByTypeNode.NAME); + if (rootDsNode != null) { + for (Node dataSourceLevelNode : getDataSourceLevelNodes(rootDsNode)) { + DataSource dataSource = dataSourceLevelNode.getLookup().lookup(DataSource.class); + if (dataSource != null) { + // the tree view needs to be searched to find the parent treeview node. + Node potentialParentTreeViewNode = findParentNodeInTree(parentContent, dataSourceLevelNode); + if (potentialParentTreeViewNode != null) { + parentTreeViewNode = potentialParentTreeViewNode; + break; + } + } } } } @@ -286,6 +301,33 @@ public class ViewContextAction extends AbstractAction { }); } + /** + * If the node has lookup of host or person, returns children. If not, just + * returns itself. + * + * @param node The node. + * @return The child nodes that are at the data source level. + */ + private List getDataSourceLevelNodes(Node node) { + if (node == null) { + return Collections.emptyList(); + } else if (node.getLookup().lookup(Host.class) != null || + node.getLookup().lookup(Person.class) != null || + DataSourcesByTypeNode.NAME.equals(node.getLookup().lookup(String.class))) { + Children children = node.getChildren(); + Node[] childNodes = children == null ? null : children.getNodes(); + if (childNodes == null) { + return Collections.emptyList(); + } + + return Stream.of(node.getChildren().getNodes()) + .flatMap(parent -> getDataSourceLevelNodes(parent).stream()) + .collect(Collectors.toList()); + } else { + return Arrays.asList(node); + } + } + /** * Searches tree for parent node by getting an ordered list of the ancestors * of the specified content. From aa41747841d943db595098f4b4a5cdd8ff8ede7b Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 23 Feb 2021 20:54:13 -0500 Subject: [PATCH 11/37] headers for host --- .../datamodel/Bundle.properties-MERGED | 2 +- .../sleuthkit/autopsy/datamodel/HostNode.java | 35 ++++++++++++---- .../autopsy/datamodel/PersonGroupingNode.java | 41 +++++++++++++------ 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED index d4ff918e9a..c3f0871cf4 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED @@ -303,7 +303,6 @@ OpenReportAction.actionPerformed.NoAssociatedEditorMessage=There is no associate OpenReportAction.actionPerformed.NoOpenInEditorSupportMessage=This platform (operating system) does not support opening a file in an editor this way. OpenReportAction.actionPerformed.MissingReportFileMessage=The report file no longer exists. OpenReportAction.actionPerformed.ReportFileOpenPermissionDeniedMessage=Permission to open the report file was denied. -PersonGroupingNode_createSheet_nameProperty=Name OsAccount_listNode_name=OS Accounts OsAccounts_accountNameProperty_desc=Os Account name OsAccounts_accountNameProperty_displayName=Name @@ -317,6 +316,7 @@ OsAccounts_createdTimeProperty_name=creationTime OsAccounts_loginNameProperty_desc=Os Account login name OsAccounts_loginNameProperty_displayName=Login Name OsAccounts_loginNameProperty_name=loginName +PersonGroupingNode_createSheet_nameProperty=Name PersonNode_unknownPersonNode_title=Unknown Persons PoolNode.createSheet.name.desc=no description PoolNode.createSheet.name.displayName=Name diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/HostNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/HostNode.java index 2fee291117..4d5b2696be 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/HostNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/HostNode.java @@ -20,8 +20,6 @@ package org.sleuthkit.autopsy.datamodel; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; -import java.util.Arrays; -import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.function.Function; @@ -159,6 +157,18 @@ public class HostNode extends DisplayableItemNode { return new DataSourceGroupingNode(key.getDataSource()); }; + /** + * Get the host name or 'unknown host' if null. + * + * @param host The host. + * @return The display name. + */ + private static String getHostName(Host host) { + return (host == null || host.getName() == null) + ? Bundle.HostGroupingNode_unknownHostNode_title() + : host.getName(); + } + private final Host host; /** @@ -188,14 +198,21 @@ public class HostNode extends DisplayableItemNode { * @param host The host. */ private HostNode(Children children, Host host) { - super(children, host == null ? null : Lookups.singleton(host)); + this(children, host, getHostName(host)); + } - String safeName = (host == null || host.getName() == null) - ? Bundle.HostGroupingNode_unknownHostNode_title() - : host.getName(); - - super.setName(safeName); - super.setDisplayName(safeName); + /** + * Constructor. + * + * @param children The children for this host node. + * @param host The host. + * @param displayName The displayName. + */ + private HostNode(Children children, Host host, String displayName) { + super(children, + host == null ? Lookups.fixed(displayName) : Lookups.fixed(host, displayName)); + super.setName(displayName); + super.setDisplayName(displayName); this.setIconBaseWithExtension(ICON_PATH); this.host = host; } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java index 7918a84a79..f788e100ab 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java @@ -147,20 +147,38 @@ public class PersonGroupingNode extends DisplayableItemNode { } } + /** + * Gets the display name for this person or "Unknown Persons". + * + * @param person The person. + * @return The non-empty string for the display name. + */ + private static String getDisplayName(Person person) { + return (person == null || person.getName() == null) + ? Bundle.PersonNode_unknownPersonNode_title() + : person.getName(); + } + /** * Main constructor. * * @param person The person record to be represented. */ PersonGroupingNode(Person person) { - super(Children.create(new PersonChildren(person), false), person == null ? null : Lookups.singleton(person)); + this(person, getDisplayName(person)); + } - String safeName = (person == null || person.getName() == null) - ? Bundle.PersonNode_unknownPersonNode_title() - : person.getName(); - - super.setName(safeName); - super.setDisplayName(safeName); + /** + * Constructor. + * + * @param person The person. + * @param displayName The display name for the person. + */ + private PersonGroupingNode(Person person, String displayName) { + super(Children.create(new PersonChildren(person), false), + person == null ? Lookups.fixed(displayName) : Lookups.fixed(person, displayName)); + super.setName(displayName); + super.setDisplayName(displayName); this.setIconBaseWithExtension(ICON_PATH); } @@ -178,11 +196,10 @@ public class PersonGroupingNode extends DisplayableItemNode { public T accept(DisplayableItemNodeVisitor visitor) { return visitor.visit(this); } - + @NbBundle.Messages({ - "PersonGroupingNode_createSheet_nameProperty=Name", - }) - @Override + "PersonGroupingNode_createSheet_nameProperty=Name",}) + @Override protected Sheet createSheet() { Sheet sheet = Sheet.createDefault(); Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); @@ -192,7 +209,7 @@ public class PersonGroupingNode extends DisplayableItemNode { } sheetSet.put(new NodeProperty<>("Name", Bundle.PersonGroupingNode_createSheet_nameProperty(), "", getDisplayName())); //NON-NLS - + return sheet; } } From 98bf69a7883d3d179b093df694d576bb6cf8ac0e Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 24 Feb 2021 09:56:06 -0500 Subject: [PATCH 12/37] getNameIdentifier --- .../autopsy/datamodel/DataSourcesByTypeNode.java | 9 ++++++++- .../sleuthkit/autopsy/datamodel/DataSourcesNode.java | 10 +++++++++- .../autopsy/directorytree/ViewContextAction.java | 8 ++++---- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java index 3152573403..e855fa8855 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java @@ -93,8 +93,15 @@ public class DataSourcesByTypeNode extends DisplayableItemNode { } - public static final String NAME = Bundle.DataSourcesHostsNode_name(); + private static final String NAME = Bundle.DataSourcesHostsNode_name(); + /** + * @return The name used to identify the node of this type with a lookup. + */ + public static String getNameIdentifier() { + return NAME; + } + /** * Main constructor. */ diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesNode.java index 5ab29a5376..033f14fc04 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesNode.java @@ -43,7 +43,15 @@ import org.sleuthkit.datamodel.TskDataException; */ public class DataSourcesNode extends DisplayableItemNode { - public static final String NAME = NbBundle.getMessage(DataSourcesNode.class, "DataSourcesNode.name"); + private static final String NAME = NbBundle.getMessage(DataSourcesNode.class, "DataSourcesNode.name"); + + /** + * @return The name used to identify the node of this type with a lookup. + */ + public static String getNameIdentifier() { + return NAME; + } + private final String displayName; // NOTE: The images passed in via argument will be ignored. diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java index 7a105f389c..cd66b54971 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java @@ -211,7 +211,7 @@ public class ViewContextAction extends AbstractAction { } // for this data source, get the "Data Sources" child node - Node datasourceGroupingNode = treeNode.getChildren().findChild(DataSourcesNode.NAME); + Node datasourceGroupingNode = treeNode.getChildren().findChild(DataSourcesNode.getNameIdentifier()); // check whether this is the data source we are looking for parentTreeViewNode = findParentNodeInTree(parentContent, datasourceGroupingNode); @@ -227,7 +227,7 @@ public class ViewContextAction extends AbstractAction { Node datasourceGroupingNode = rootChildren.findChild(dsname); if (!Objects.isNull(datasourceGroupingNode)) { Children dsChildren = datasourceGroupingNode.getChildren(); - parentTreeViewNode = dsChildren.findChild(DataSourcesNode.NAME); + parentTreeViewNode = dsChildren.findChild(DataSourcesNode.getNameIdentifier()); } } @@ -244,7 +244,7 @@ public class ViewContextAction extends AbstractAction { } else { // Classic view // Start the search at the DataSourcesNode Children rootChildren = treeViewExplorerMgr.getRootContext().getChildren(); - Node rootDsNode = rootChildren == null ? null : rootChildren.findChild(DataSourcesByTypeNode.NAME); + Node rootDsNode = rootChildren == null ? null : rootChildren.findChild(DataSourcesByTypeNode.getNameIdentifier()); if (rootDsNode != null) { for (Node dataSourceLevelNode : getDataSourceLevelNodes(rootDsNode)) { DataSource dataSource = dataSourceLevelNode.getLookup().lookup(DataSource.class); @@ -313,7 +313,7 @@ public class ViewContextAction extends AbstractAction { return Collections.emptyList(); } else if (node.getLookup().lookup(Host.class) != null || node.getLookup().lookup(Person.class) != null || - DataSourcesByTypeNode.NAME.equals(node.getLookup().lookup(String.class))) { + DataSourcesByTypeNode.getNameIdentifier().equals(node.getLookup().lookup(String.class))) { Children children = node.getChildren(); Node[] childNodes = children == null ? null : children.getNodes(); if (childNodes == null) { From 76cf9fa2010dadd593bcbdd16a13c42735f71d87 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 24 Feb 2021 15:10:11 -0500 Subject: [PATCH 13/37] Bug fixes and code polish --- .../autopsy/keywordsearch/Bundle.properties | 4 +--- .../keywordsearch/Bundle.properties-MERGED | 15 +++------------ .../autopsy/keywordsearch/Bundle_ja.properties | 3 +-- .../keywordsearch/ExtractAllTermsReport.java | 14 ++++++-------- .../org/sleuthkit/autopsy/keywordsearch/layer.xml | 4 ---- 5 files changed, 11 insertions(+), 29 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 8a0054c80d..d2ab5f76c7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -23,7 +23,7 @@ KeywordSearchEditListPanel.emptyKeyword.text=Empty keyword KeywordSearchEditListPanel.errorAddingKeywords.text=Error adding keyword(s) KeywordSearchListsManagementPanel.newListButton.text=New List KeywordSearchListsManagementPanel.importButton.text=Import List -KeywordSearchListsViewerPanel.searchAddButton.text=Extract +KeywordSearchListsViewerPanel.searchAddButton.text=Search KeywordSearchListsViewerPanel.manageListsButton.text=Manage Lists KeywordSearchListsViewerPanel.ingestIndexLabel.text=Files Indexed: KeywordSearchEditListPanel.ingestMessagesCheckbox.text=Send ingest inbox messages for each hit @@ -320,5 +320,3 @@ ExtractedContentPanel.pageOfLabel.text=of ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) -TermsExtractionDialog.selectDataSourceLabel.text=Select Data Source: -TermsExtractionDialog.statusLabel.text= diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index e3f51bbb12..7856cd5bb4 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -10,17 +10,18 @@ DropdownSingleTermSearchPanel.selected=Ad Hoc Search data source filter is selec DropdownSingleTermSearchPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] DropdownSingleTermSearchPanel.warning.title=Warning ExtractAllTermsReport.error.noOpenCase=No currently open case. -ExtractAllTermsReport.error.unableToOpenCase=Exception while getting open case. ExtractAllTermsReport.export.error=Error During Unique Word Export ExtractAllTermsReport.exportComplete=Unique Word Export Complete ExtractAllTermsReport.getName.text=Extract Unique Words ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms... ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway? +# {0} - Keyword search commit frequency ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes. ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet.
Try again later ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress ExtractAllTermsReport.startExport=Starting Unique Word Export ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait +# {0} - Content name ExtractedContentPanel.SetMarkup.progress.loading=Loading text for {0} GlobalEditListPanel.editKeyword.title=Edit Keyword GlobalEditListPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] @@ -70,7 +71,7 @@ KeywordSearchEditListPanel.emptyKeyword.text=Empty keyword KeywordSearchEditListPanel.errorAddingKeywords.text=Error adding keyword(s) KeywordSearchListsManagementPanel.newListButton.text=New List KeywordSearchListsManagementPanel.importButton.text=Import List -KeywordSearchListsViewerPanel.searchAddButton.text=Extract +KeywordSearchListsViewerPanel.searchAddButton.text=Search KeywordSearchListsViewerPanel.manageListsButton.text=Manage Lists KeywordSearchListsViewerPanel.ingestIndexLabel.text=Files Indexed: KeywordSearchEditListPanel.ingestMessagesCheckbox.text=Send ingest inbox messages for each hit @@ -398,18 +399,8 @@ ExtractedContentPanel.pageOfLabel.text=of ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) -TermsExtractionDialog.dialogErrorHeader=Error Extracting Unique Words -TermsExtractionDialog.export.error=Error During Unique Word Export -TermsExtractionDialog.exportComplete=Unique Word Export Complete -TermsExtractionDialog.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway? -TermsExtractionDialog.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes. -TermsExtractionDialog.search.noFilesInIdxMsg2=No files are in index yet.
Try again later -TermsExtractionDialog.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress TermsExtractionDialog.selectDataSourceLabel.text=Select Data Source: -TermsExtractionDialog.startExport=Starting Unique Word Export TermsExtractionDialog.statusLabel.text= -TermsExtractionDialog.unableToGetDataSources.error=Unable to populate list of data sources -TermsExtractionPanel.selected=Data source filter is selected TextZoomPanel.zoomInButton.text= TextZoomPanel.zoomOutButton.text= TextZoomPanel.zoomResetButton.text=Reset diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties index cf7f6f92cb..2f50e62095 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties @@ -398,5 +398,4 @@ ExtractedContentPanel.pageOfLabel.text=/ ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=\u30da\u30fc\u30b8: TextZoomPanel.zoomResetButton.text=\u30ea\u30bb\u30c3\u30c8 -TermsExtractionDialog.selectDataSourceLabel.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb: -TermsExtractionDialog.statusLabel.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb: + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java index b1aa714af7..40da04822c 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java @@ -49,8 +49,9 @@ public class ExtractAllTermsReport implements GeneralReportModule { @NbBundle.Messages({ "ExtractAllTermsReport.error.noOpenCase=No currently open case.", - "ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes.", - "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet.
Try again later", + "# {0} - Keyword search commit frequency", + "ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. Try again later. Index is updated every {0} minutes.", + "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try again later", "ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress", "ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway?", "ExtractAllTermsReport.startExport=Starting Unique Word Export", @@ -85,7 +86,6 @@ public class ExtractAllTermsReport implements GeneralReportModule { progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_search_noFilesInIdxMsg2()); } progressPanel.setIndeterminate(false); - progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR); return; } @@ -105,16 +105,14 @@ public class ExtractAllTermsReport implements GeneralReportModule { Path outputFile = Paths.get(settings.getReportDirectoryPath(), getRelativeFilePath()); server.extractAllTermsForDataSource(outputFile, progressPanel); } catch (KeywordSearchModuleException | NoOpenCoreException ex) { - logger.log(Level.SEVERE, "Exception while extracting all terms", ex); //NON-NLS - progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_export_error()); + logger.log(Level.SEVERE, "Exception while extracting unique terms", ex); //NON-NLS progressPanel.setIndeterminate(false); - progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR); + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_export_error()); return; } - progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_exportComplete()); progressPanel.setIndeterminate(false); - progressPanel.complete(ReportProgressPanel.ReportStatus.COMPLETE); + progressPanel.complete(ReportProgressPanel.ReportStatus.COMPLETE, Bundle.ExtractAllTermsReport_exportComplete()); } @Override diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/layer.xml b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/layer.xml index 8effaba623..bde41b641f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/layer.xml +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/layer.xml @@ -7,10 +7,6 @@ ====================================================== --> - - - - From b5849851343594aea708bf8eac1c977d15d4a23f Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 24 Feb 2021 20:48:32 -0500 Subject: [PATCH 14/37] view source result fix --- .../autopsy/datamodel/ResultsNode.java | 6 +- .../DirectoryTreeTopComponent.java | 92 +++++++++++++++++-- 2 files changed, 90 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ResultsNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/ResultsNode.java index 339c6bb29b..e9b2eba696 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ResultsNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ResultsNode.java @@ -31,8 +31,12 @@ import org.sleuthkit.datamodel.SleuthkitCase; public class ResultsNode extends DisplayableItemNode { @NbBundle.Messages("ResultsNode.name.text=Results") - public static final String NAME = Bundle.ResultsNode_name_text(); + private static final String NAME = Bundle.ResultsNode_name_text(); + public static String getNameIdentifier() { + return NAME; + } + public ResultsNode(SleuthkitCase sleuthkitCase) { this(sleuthkitCase, 0); } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index b4c9e29658..e616b4ac9c 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -35,6 +35,7 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.prefs.PreferenceChangeEvent; import java.util.prefs.PreferenceChangeListener; +import java.util.stream.Stream; import javax.swing.Action; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; @@ -81,6 +82,8 @@ import org.sleuthkit.autopsy.datamodel.InterestingHits; import org.sleuthkit.autopsy.datamodel.KeywordHits; import org.sleuthkit.autopsy.datamodel.ResultsNode; import org.sleuthkit.autopsy.datamodel.AutopsyTreeChildFactory; +import org.sleuthkit.autopsy.datamodel.DataSourceGrouping; +import org.sleuthkit.autopsy.datamodel.DataSourcesByTypeNode; import org.sleuthkit.autopsy.datamodel.Tags; import org.sleuthkit.autopsy.datamodel.ViewsNode; import org.sleuthkit.autopsy.datamodel.accounts.Accounts; @@ -89,6 +92,9 @@ import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.Host; +import org.sleuthkit.datamodel.Person; import org.sleuthkit.datamodel.TskCoreException; /** @@ -193,7 +199,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private void preExpandNodes(Children rootChildren) { BeanTreeView tree = getTree(); - Node results = rootChildren.findChild(ResultsNode.NAME); + Node results = rootChildren.findChild(ResultsNode.getNameIdentifier()); if (!Objects.isNull(results)) { tree.expandNode(results); Children resultsChildren = results.getChildren(); @@ -265,7 +271,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * Setter to determine if rejected results should be shown or not. * * @param showRejectedResults True if showing rejected results; otherwise - * false. + * false. */ public void setShowRejectedResults(boolean showRejectedResults) { this.showRejectedResults = showRejectedResults; @@ -797,7 +803,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } // change in node selection else if (changed.equals(ExplorerManager.PROP_SELECTED_NODES)) { respondSelection((Node[]) event.getOldValue(), (Node[]) event.getNewValue()); - } + } } } @@ -1012,8 +1018,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * Set the selected node using a path to a previously selected node. * * @param previouslySelectedNodePath Path to a previously selected node. - * @param rootNodeName Name of the root node to match, may be - * null. + * @param rootNodeName Name of the root node to match, may be null. */ private void setSelectedNode(final String[] previouslySelectedNodePath, final String rootNodeName) { if (previouslySelectedNodePath == null) { @@ -1070,12 +1075,85 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat return false; } + /** + * Does dfs search of node while nodes are Host, Person, or + * DataSourcesByType looking for the Results Node. + * + * @param node The node. + * @return The child nodes that are at the data source level. + */ + private Node getResultsNodeSearch(Node node, long dataSourceId) { + if (node == null) { + return null; + } else if (node.getLookup().lookup(Host.class) != null + || node.getLookup().lookup(Person.class) != null) { + Children children = node.getChildren(); + Node[] childNodes = children == null ? null : children.getNodes(); + if (childNodes != null) { + for (Node child : childNodes) { + Node foundExtracted = getResultsNodeSearch(child, dataSourceId); + if (foundExtracted != null) { + return foundExtracted; + } + } + } + } else { + DataSource dataSource = node.getLookup().lookup(DataSource.class); + if (dataSource != null && dataSource.getId() == dataSourceId) { + Children dsChildren = node.getChildren(); + if (dsChildren != null) { + return dsChildren.findChild(ResultsNode.getNameIdentifier()); + } + } + } + return null; + } + + /** + * Finds the results node for the specific artifact. + * + * @param art The artifact to find the relevant Results Node. + * @return THe Results Node or null. + */ + private Node getResultsNode(final BlackboardArtifact art) { + Children rootChilds = em.getRootContext().getChildren(); + + Node resultsNode = rootChilds.findChild(ResultsNode.getNameIdentifier()); + if (resultsNode != null) { + return resultsNode; + } + + long dataSourceId; + try { + dataSourceId = art.getDataSource().getId(); + } catch (TskCoreException ex) { + LOGGER.log(Level.WARNING, "There was an error fetching the data source id for artifact.", ex); + return null; + } + + Node[] rootNodes = rootChilds.getNodes(); + if (rootNodes != null) { + for (Node rootNode : rootNodes) { + resultsNode = getResultsNodeSearch(rootNode, dataSourceId); + if (resultsNode != null) { + return resultsNode; + } + } + } + + return null; + } + public void viewArtifact(final BlackboardArtifact art) { int typeID = art.getArtifactTypeID(); String typeName = art.getArtifactTypeName(); - Children rootChilds = em.getRootContext().getChildren(); Node treeNode = null; - Node resultsNode = rootChilds.findChild(ResultsNode.NAME); + + Node resultsNode = getResultsNode(art); + if (resultsNode == null) { + return; + } + Children resultsChilds = resultsNode.getChildren(); if (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID()) { Node hashsetRootNode = resultsChilds.findChild(typeName); From 8e214d34954f2143af382081bd277bd6be2ffa39 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 25 Feb 2021 14:29:19 -0500 Subject: [PATCH 15/37] Bug fixes --- .../autopsy/keywordsearch/Bundle.properties-MERGED | 8 +++----- .../autopsy/keywordsearch/ExtractAllTermsReport.java | 5 ++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 7856cd5bb4..edff9a8e5e 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -16,8 +16,8 @@ ExtractAllTermsReport.getName.text=Extract Unique Words ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms... ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway? # {0} - Keyword search commit frequency -ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet.
Try again later. Index is updated every {0} minutes. -ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet.
Try again later +ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. Try again later. Index is updated every {0} minutes. +ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try again later ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress ExtractAllTermsReport.startExport=Starting Unique Word Export ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait @@ -48,7 +48,7 @@ KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search @@ -399,8 +399,6 @@ ExtractedContentPanel.pageOfLabel.text=of ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) -TermsExtractionDialog.selectDataSourceLabel.text=Select Data Source: -TermsExtractionDialog.statusLabel.text= TextZoomPanel.zoomInButton.text= TextZoomPanel.zoomOutButton.text= TextZoomPanel.zoomResetButton.text=Reset diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java index 40da04822c..fa12b5747b 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java @@ -53,7 +53,7 @@ public class ExtractAllTermsReport implements GeneralReportModule { "ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. Try again later. Index is updated every {0} minutes.", "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try again later", "ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress", - "ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway?", + "ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and unique word extraction might yield incomplete results.
Do you want to proceed with unique word extraction anyway?", "ExtractAllTermsReport.startExport=Starting Unique Word Export", "ExtractAllTermsReport.export.error=Error During Unique Word Export", "ExtractAllTermsReport.exportComplete=Unique Word Export Complete" @@ -93,8 +93,7 @@ public class ExtractAllTermsReport implements GeneralReportModule { if (isIngestRunning) { if (KeywordSearchUtil.displayConfirmDialog(Bundle.ExtractAllTermsReport_search_searchIngestInProgressTitle(), Bundle.ExtractAllTermsReport_search_ingestInProgressBody(), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN) == false) { - progressPanel.setIndeterminate(false); - progressPanel.complete(ReportProgressPanel.ReportStatus.CANCELED); + progressPanel.cancel(); return; } } From ffa9a3e4fed7101bff273756d8f72e4560a444f8 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 25 Feb 2021 14:52:59 -0500 Subject: [PATCH 16/37] Code polish --- .../keywordsearch/Bundle.properties-MERGED | 13 +++++++------ .../keywordsearch/ExtractAllTermsReport.java | 10 ++++++---- .../autopsy/keywordsearch/Server.java | 18 +++++++++++++++--- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index edff9a8e5e..72de1c17d8 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -9,19 +9,20 @@ DropdownListSearchPanel.selected=Ad Hoc Search data source filter is selected DropdownSingleTermSearchPanel.selected=Ad Hoc Search data source filter is selected DropdownSingleTermSearchPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] DropdownSingleTermSearchPanel.warning.title=Warning +ExtractAllTermsReport.description.text=Extracts all unique words out of the current case. NOTE: The extracted words are lower-cased. ExtractAllTermsReport.error.noOpenCase=No currently open case. -ExtractAllTermsReport.export.error=Error During Unique Word Export -ExtractAllTermsReport.exportComplete=Unique Word Export Complete +ExtractAllTermsReport.export.error=Error During Unique Word Extraction +ExtractAllTermsReport.exportComplete=Unique Word Extraction Complete ExtractAllTermsReport.getName.text=Extract Unique Words +# {0} - Number of extracted terms ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms... -ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and this search might yield incomplete results.
Do you want to proceed with this search anyway? +ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and unique word extraction might yield incomplete results.
Do you want to proceed with unique word extraction anyway? # {0} - Keyword search commit frequency ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. Try again later. Index is updated every {0} minutes. ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try again later ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress -ExtractAllTermsReport.startExport=Starting Unique Word Export +ExtractAllTermsReport.startExport=Starting Unique Word Extraction ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait -# {0} - Content name ExtractedContentPanel.SetMarkup.progress.loading=Loading text for {0} GlobalEditListPanel.editKeyword.title=Edit Keyword GlobalEditListPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] @@ -48,7 +49,7 @@ KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java index fa12b5747b..f646b0863a 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java @@ -54,9 +54,9 @@ public class ExtractAllTermsReport implements GeneralReportModule { "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try again later", "ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress", "ExtractAllTermsReport.search.ingestInProgressBody=Keyword Search Ingest is currently running.
Not all files have been indexed and unique word extraction might yield incomplete results.
Do you want to proceed with unique word extraction anyway?", - "ExtractAllTermsReport.startExport=Starting Unique Word Export", - "ExtractAllTermsReport.export.error=Error During Unique Word Export", - "ExtractAllTermsReport.exportComplete=Unique Word Export Complete" + "ExtractAllTermsReport.startExport=Starting Unique Word Extraction", + "ExtractAllTermsReport.export.error=Error During Unique Word Extraction", + "ExtractAllTermsReport.exportComplete=Unique Word Extraction Complete" }) @Override public void generateReport(GeneralReportSettings settings, ReportProgressPanel progressPanel) { @@ -119,9 +119,11 @@ public class ExtractAllTermsReport implements GeneralReportModule { return false; } + @NbBundle.Messages({ + "ExtractAllTermsReport.description.text=Extracts all unique words out of the current case. NOTE: The extracted words are lower-cased."}) @Override public String getDescription() { - return "Extracts all unique words out of the current case. NOTE: The extracted words are lower-cased."; + return Bundle.ExtractAllTermsReport_description_text(); } @Override diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 2772776ed8..851e42e2da 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -1789,9 +1789,9 @@ public class Server { } /** - * Extract all unique terms/words for a given data source. + * Extract all unique terms/words from current index. * - * @param outputFile Absolute path to where the store the output + * @param outputFile Absolute path to the output file * @param progressPanel ReportProgressPanel to update * * @throws NoOpenCoreException @@ -2163,15 +2163,27 @@ public class Server { queryClient.deleteByQuery(deleteQuery); } + /** + * Extract all unique terms/words from current index. Gets 1,000 terms at a time and + * writes them to output file. Updates ReportProgressPanel status. + * + * @param outputFile Absolute path to the output file + * @param progressPanel ReportProgressPanel to update + * @throws IOException + * @throws SolrServerException + * @throws NoCurrentCaseException + * @throws KeywordSearchModuleException + */ @NbBundle.Messages({ + "# {0} - Number of extracted terms", "ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms..." }) private void extractAllTermsForDataSource(Path outputFile, ReportProgressPanel progressPanel) throws IOException, SolrServerException, NoCurrentCaseException, KeywordSearchModuleException { - // step through the terms Files.deleteIfExists(outputFile); OpenOption[] options = new OpenOption[] { java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND }; + // step through the terms int termStep = 1000; long numExtractedTerms = 0; String firstTerm = ""; From f7696486f2907778cb19ea26d85b1e4ede076c50 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 25 Feb 2021 14:54:33 -0500 Subject: [PATCH 17/37] Minor --- .../autopsy/keywordsearch/ExtractAllTermsReport.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java index f646b0863a..b3a754d6b8 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java @@ -74,7 +74,7 @@ public class ExtractAllTermsReport implements GeneralReportModule { boolean isIngestRunning = IngestManager.getInstance().isIngestRunning(); int filesIndexed = 0; - try { // see if another node added any indexed files + try { // see if there are any indexed files filesIndexed = KeywordSearch.getServer().queryNumIndexedFiles(); } catch (KeywordSearchModuleException | NoOpenCoreException ignored) { } @@ -89,7 +89,7 @@ public class ExtractAllTermsReport implements GeneralReportModule { return; } - //check if keyword search module ingest is running (indexing, etc) + // check if keyword search module ingest is running (indexing, etc) if (isIngestRunning) { if (KeywordSearchUtil.displayConfirmDialog(Bundle.ExtractAllTermsReport_search_searchIngestInProgressTitle(), Bundle.ExtractAllTermsReport_search_ingestInProgressBody(), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN) == false) { From 564fefdba8b36ba8d28dbf22332055d5db567643 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 25 Feb 2021 21:26:10 -0500 Subject: [PATCH 18/37] fix for interesting file hits --- .../directorytree/DirectoryTreeTopComponent.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index e616b4ac9c..ee83cf3e29 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -1238,11 +1238,16 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat if (setNode == null) { return; } - Children interestingChildren = setNode.getChildren(); - if (interestingChildren == null) { + + Children fileArtifactChildren = setNode.getChildren(); + Node[] fileArtifactNodes = fileArtifactChildren == null ? null : fileArtifactChildren.getNodes(); + if (fileArtifactNodes == null || fileArtifactNodes.length != 2) { return; } - treeNode = interestingChildren.findChild(art.getDisplayName()); + + treeNode = (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT.getTypeID()) ? + fileArtifactNodes[0] : + fileArtifactNodes[1]; } catch (TskCoreException ex) { LOGGER.log(Level.WARNING, "Error retrieving attributes", ex); //NON-NLS } From 2bcade64f487da102df3fc0da69e55b67c0104b7 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 26 Feb 2021 11:13:45 -0500 Subject: [PATCH 19/37] commenting --- .../DirectoryTreeTopComponent.java | 21 ++++++++++++++----- .../directorytree/ViewContextAction.java | 6 ++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index ee83cf3e29..4a3a432aed 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -1144,6 +1144,17 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat return null; } + /** + * Navigates to artifact and shows in view. + * + * NOTE: This code will likely need updating in the event that the structure + * of the nodes is changed (i.e. adding parent levels). Places to look when + * changing node structure include: + * + * DirectoryTreeTopComponent.viewArtifact, ViewContextAction + * + * @param art The artifact. + */ public void viewArtifact(final BlackboardArtifact art) { int typeID = art.getArtifactTypeID(); String typeName = art.getArtifactTypeName(); @@ -1238,16 +1249,16 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat if (setNode == null) { return; } - + Children fileArtifactChildren = setNode.getChildren(); Node[] fileArtifactNodes = fileArtifactChildren == null ? null : fileArtifactChildren.getNodes(); if (fileArtifactNodes == null || fileArtifactNodes.length != 2) { return; } - - treeNode = (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT.getTypeID()) ? - fileArtifactNodes[0] : - fileArtifactNodes[1]; + + treeNode = (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT.getTypeID()) + ? fileArtifactNodes[0] + : fileArtifactNodes[1]; } catch (TskCoreException ex) { LOGGER.log(Level.WARNING, "Error retrieving attributes", ex); //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java index cd66b54971..bb8fd7c561 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java @@ -147,6 +147,12 @@ public class ViewContextAction extends AbstractAction { * branch of the tree view to the level of the parent of the content, * selecting the parent in the tree view, then selecting the content in the * results view. + * + * NOTE: This code will likely need updating in the event that the structure + * of the nodes is changed (i.e. adding parent levels). Places to look when + * changing node structure include: + * + * DirectoryTreeTopComponent.viewArtifact, ViewContextAction * * @param event The action event. */ From 1c3059d76c7095d3d0273b564eb2261f85ffbc0a Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 1 Mar 2021 15:47:12 -0500 Subject: [PATCH 20/37] added hosts and persons autopsy events --- .../sleuthkit/autopsy/casemodule/Case.java | 159 +++++++++++++++++- .../casemodule/events/HostAddedEvent.java | 39 +++++ .../casemodule/events/HostChangedEvent.java | 41 +++++ .../autopsy/casemodule/events/HostEvent.java | 87 ++++++++++ .../casemodule/events/HostRemovedEvent.java | 39 +++++ .../casemodule/events/PersonAddedEvent.java | 39 +++++ .../casemodule/events/PersonChangedEvent.java | 41 +++++ .../casemodule/events/PersonEvent.java | 87 ++++++++++ .../casemodule/events/PersonRemovedEvent.java | 39 +++++ 9 files changed, 569 insertions(+), 2 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/events/HostAddedEvent.java create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/events/HostChangedEvent.java create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/events/HostEvent.java create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/events/HostRemovedEvent.java create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/events/PersonAddedEvent.java create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/events/PersonChangedEvent.java create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/events/PersonEvent.java create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/events/PersonRemovedEvent.java diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 0115b3a53c..eb37895a30 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -39,6 +39,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.text.SimpleDateFormat; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -81,6 +82,9 @@ import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; import org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent; import org.sleuthkit.autopsy.casemodule.events.DataSourceDeletedEvent; import org.sleuthkit.autopsy.casemodule.events.DataSourceNameChangedEvent; +import org.sleuthkit.autopsy.casemodule.events.HostAddedEvent; +import org.sleuthkit.autopsy.casemodule.events.HostChangedEvent; +import org.sleuthkit.autopsy.casemodule.events.HostRemovedEvent; import org.sleuthkit.autopsy.casemodule.events.OsAccountAddedEvent; import org.sleuthkit.autopsy.casemodule.events.OsAccountChangedEvent; import org.sleuthkit.autopsy.casemodule.events.ReportAddedEvent; @@ -130,6 +134,7 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.FileSystem; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.OsAccount; import org.sleuthkit.datamodel.OsAccountManager; @@ -426,8 +431,38 @@ public class Case { * OSAccount associated with the current case has changed. * Call getOsAccount to get the changed account; */ - OS_ACCOUNT_CHANGED; + OS_ACCOUNT_CHANGED, + + /** + * Hosts associated with the current case added. + */ + HOSTS_ADDED, + /** + * Hosts associated with the current case has changed. + */ + HOSTS_CHANGED, + + /** + * Hosts associated with the current case has been deleted. + */ + HOSTS_DELETED, + + /** + * Persons associated with the current case added. + */ + PERSONS_ADDED, + + /** + * Persons associated with the current case has changed. + */ + PERSONS_CHANGED, + + /** + * Persons associated with the current case has been deleted. + */ + PERSONS_DELETED + ; }; /** @@ -474,6 +509,78 @@ public class Case { eventPublisher.publish(new OsAccountChangedEvent(account)); } } + + /** + * Publishes an autopsy event from the sleuthkit HostCreationEvent + * indicating that hosts have been created. + * + * @param event The sleuthkit event for the creation of hosts. + */ + @Subscribe + public void publishHostsAddedEvent(HostCreationEvent event) { + eventPublisher.publish(new HostsAddedEvent( + event == null ? Collections.emptyList() : event.getHosts())); + } + + /** + * Publishes an autopsy event from the sleuthkit HostUpdateEvent + * indicating that hosts have been updated. + * + * @param event The sleuthkit event for the updating of hosts. + */ + @Subscribe + public void publishHostsChangedEvent(HostUpdateEvent event) { + eventPublisher.publish(new HostsChangedEvent( + event == null ? Collections.emptyList() : event.getHosts())); + } + + /** + * Publishes an autopsy event from the sleuthkit HostDeletedEvent + * indicating that hosts have been deleted. + * + * @param event The sleuthkit event for the deleting of hosts. + */ + @Subscribe + public void publishHostsDeletedEvent(HostDeletedEvent event) { + eventPublisher.publish(new HostsRemovedEvent( + event == null ? Collections.emptyList() : event.getHosts())); + } + + /** + * Publishes an autopsy event from the sleuthkit PersonCreationEvent + * indicating that persons have been created. + * + * @param event The sleuthkit event for the creation of persons. + */ + @Subscribe + public void publishPersonsAddedEvent(PersonCreationEvent event) { + eventPublisher.publish(new PersonsAddedEvent( + event == null ? Collections.emptyList() : event.getPersons())); + } + + /** + * Publishes an autopsy event from the sleuthkit PersonUpdateEvent + * indicating that persons have been updated. + * + * @param event The sleuthkit event for the updating of persons. + */ + @Subscribe + public void publishPersonsChangedEvent(PersonUpdateEvent event) { + eventPublisher.publish(new PersonsChangedEvent( + event == null ? Collections.emptyList() : event.getPersons())); + } + + /** + * Publishes an autopsy event from the sleuthkit PersonDeletedEvent + * indicating that persons have been deleted. + * + * @param event The sleuthkit event for the deleting of persons. + */ + @Subscribe + public void publishPersonsDeletedEvent(PersonDeletedEvent event) { + eventPublisher.publish(new PersonsRemovedEvent( + event == null ? Collections.emptyList() : event.getPersons())); + } } /** @@ -1680,7 +1787,55 @@ public class Case { public void notifyOsAccountChanged(OsAccount account) { eventPublisher.publish(new OsAccountChangedEvent(account)); } - + + /** + * Notify via an autopsy event that a host has been added. + * @param host The host that has been added. + */ + public void notifyHostAdded(Host host) { + eventPublisher.publish(new HostAddedEvent(Collections.singletonList(host))); + } + + /** + * Notify via an autopsy event that a host has been changed. + * @param newValue The host that has been updated. + */ + public void notifyHostChanged(Host newValue) { + eventPublisher.publish(new HostChangedEvent(Collections.singletonList(newValue))); + } + + /** + * Notify via an autopsy event that a host has been deleted. + * @param host The host that has been deleted. + */ + public void notifyHostDeleted(Host host) { + eventPublisher.publish(new HostRemovedEvent(Collections.singletonList(host))); + } + + /** + * Notify via an autopsy event that a person has been added. + * @param person The person that has been added. + */ + public void notifyPersonAdded(Person person) { + eventPublisher.publish(new PersonAddedEvent(Collections.singletonList(person))); + } + + /** + * Notify via an autopsy event that a person has been changed. + * @param newValue The person that has been updated. + */ + public void notifyPersonChanged(Person newValue) { + eventPublisher.publish(new PersonChangedEvent(Collections.singletonList(newValue))); + } + + /** + * Notify via an autopsy event that a person has been deleted. + * @param person The person that has been deleted. + */ + public void notifyPersonDeleted(Person person) { + eventPublisher.publish(new PersonRemovedEvent(Collections.singletonList(person))); + } + /** * Adds a report to the case. * diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostAddedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostAddedEvent.java new file mode 100644 index 0000000000..3676c475c0 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostAddedEvent.java @@ -0,0 +1,39 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule.events; + +import java.util.List; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.Host; + +/** + * Event fired when new hosts are added. + */ +public class HostAddedEvent extends HostEvent { + + private static final long serialVersionUID = 1L; + + /** + * Main constructor. + * @param dataModelObjects The hosts that have been added. + */ + public HostAddedEvent(List dataModelObjects) { + super(Case.Events.HOST_ADDED.name(), dataModelObjects); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostChangedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostChangedEvent.java new file mode 100644 index 0000000000..37e589ea61 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostChangedEvent.java @@ -0,0 +1,41 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule.events; + +import java.util.List; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.Host; + +/** + * Event fired when hosts are changed. + */ +public class HostChangedEvent extends HostEvent { + + private static final long serialVersionUID = 1L; + + /** + * Main constructor. + * + * @param dataModelObjects The new values for the hosts that have been + * changed. + */ + public HostChangedEvent(List dataModelObjects) { + super(Case.Events.HOST_CHANGED.name(), dataModelObjects); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostEvent.java new file mode 100644 index 0000000000..0f2e1ed0e4 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostEvent.java @@ -0,0 +1,87 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule.events; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.sleuthkit.datamodel.Host; +import org.sleuthkit.datamodel.HostManager; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Base event class for when something pertaining to hosts changes. + */ +public class HostEvent extends TskDataModelChangeEvent { + + /** + * Retrieves a list of ids from a list of hosts. + * + * @param hosts The hosts. + * @return The list of ids. + */ + private static List getIds(List hosts) { + return getSafeList(hosts).stream() + .filter(h -> h != null) + .map(h -> h.getId()).collect(Collectors.toList()); + } + + /** + * Returns the hosts or an empty list. + * + * @param hosts The host list. + * @return The host list or an empty list if the parameter is null. + */ + private static List getSafeList(List hosts) { + return hosts == null ? Collections.emptyList() : hosts; + } + + /** + * Main constructor. + * + * @param eventName The name of the Case.Events enum value for the event + * type. + * @param dataModelObjects The list of hosts for the event. + */ + protected HostEvent(String eventName, List dataModelObjects) { + super(eventName, getIds(dataModelObjects), new ArrayList<>(getSafeList(dataModelObjects))); + } + + @Override + protected List getDataModelObjects(SleuthkitCase caseDb, List ids) throws TskCoreException { + HostManager hostManager = caseDb.getHostManager(); + List toRet = new ArrayList<>(); + if (ids != null) { + for (Long id : ids) { + if (id == null) { + continue; + } + + Optional thisHostOpt = hostManager.getHost(id); + thisHostOpt.ifPresent((h) -> toRet.add(h)); + } + } + + return toRet; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostRemovedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostRemovedEvent.java new file mode 100644 index 0000000000..22cd237629 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostRemovedEvent.java @@ -0,0 +1,39 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule.events; + +import java.util.List; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.Host; + +/** + * Event fired when hosts are removed. + */ +public class HostRemovedEvent extends HostEvent { + + private static final long serialVersionUID = 1L; + + /** + * Main constructor. + * @param dataModelObjects The list of hosts that have been deleted. + */ + public HostRemovedEvent(List dataModelObjects) { + super(Case.Events.HOST_DELETED.name(), dataModelObjects); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonAddedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonAddedEvent.java new file mode 100644 index 0000000000..2d33315f95 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonAddedEvent.java @@ -0,0 +1,39 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule.events; + +import java.util.List; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.Person; + +/** + * Event fired when new persons are added. + */ +public class PersonAddedEvent extends PersonEvent { + + private static final long serialVersionUID = 1L; + + /** + * Main constructor. + * @param dataModelObjects The persons that have been added. + */ + public PersonAddedEvent(List dataModelObjects) { + super(Case.Events.PERSON_ADDED.name(), dataModelObjects); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonChangedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonChangedEvent.java new file mode 100644 index 0000000000..9209497de8 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonChangedEvent.java @@ -0,0 +1,41 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule.events; + +import java.util.List; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.Person; + +/** + * Event fired when persons are changed. + */ +public class PersonChangedEvent extends PersonEvent { + + private static final long serialVersionUID = 1L; + + /** + * Main constructor. + * + * @param dataModelObjects The new values for the persons that have been + * changed. + */ + public PersonChangedEvent(List dataModelObjects) { + super(Case.Events.PERSON_CHANGED.name(), dataModelObjects); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonEvent.java new file mode 100644 index 0000000000..f41ae6ff86 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonEvent.java @@ -0,0 +1,87 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule.events; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.sleuthkit.datamodel.Person; +import org.sleuthkit.datamodel.PersonManager; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Base event class for when something pertaining to persons changes. + */ +public class PersonEvent extends TskDataModelChangeEvent { + + /** + * Retrieves a list of ids from a list of persons. + * + * @param persons The persons. + * @return The list of ids. + */ + private static List getIds(List persons) { + return getSafeList(persons).stream() + .filter(h -> h != null) + .map(h -> h.getId()).collect(Collectors.toList()); + } + + /** + * Returns the persons or an empty list. + * + * @param persons The person list. + * @return The person list or an empty list if the parameter is null. + */ + private static List getSafeList(List persons) { + return persons == null ? Collections.emptyList() : persons; + } + + /** + * Main constructor. + * + * @param eventName The name of the Case.Events enum value for the event + * type. + * @param dataModelObjects The list of persons for the event. + */ + protected PersonEvent(String eventName, List dataModelObjects) { + super(eventName, getIds(dataModelObjects), new ArrayList<>(getSafeList(dataModelObjects))); + } + + @Override + protected List getDataModelObjects(SleuthkitCase caseDb, List ids) throws TskCoreException { + PersonManager personManager = caseDb.getPersonManager(); + List toRet = new ArrayList<>(); + if (ids != null) { + for (Long id : ids) { + if (id == null) { + continue; + } + + Optional thisPersonOpt = personManager.getPerson(id); + thisPersonOpt.ifPresent((h) -> toRet.add(h)); + } + } + + return toRet; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonRemovedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonRemovedEvent.java new file mode 100644 index 0000000000..a33af32cc7 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonRemovedEvent.java @@ -0,0 +1,39 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule.events; + +import java.util.List; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.Person; + +/** + * Event fired when persons are removed. + */ +public class PersonRemovedEvent extends PersonEvent { + + private static final long serialVersionUID = 1L; + + /** + * Main constructor. + * @param dataModelObjects The list of persons that have been deleted. + */ + public PersonRemovedEvent(List dataModelObjects) { + super(Case.Events.PERSON_DELETED.name(), dataModelObjects); + } +} From f718f3ae07d23d7c8a39ecd73f77b1cfd902eace Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 1 Mar 2021 20:37:57 -0500 Subject: [PATCH 21/37] updates --- .../sleuthkit/autopsy/casemodule/Case.java | 40 ++++++++++------- ...stAddedEvent.java => HostsAddedEvent.java} | 6 +-- ...angedEvent.java => HostsChangedEvent.java} | 6 +-- .../{HostEvent.java => HostsEvent.java} | 4 +- ...movedEvent.java => HostsRemovedEvent.java} | 6 +-- ...AddedEvent.java => PersonsAddedEvent.java} | 6 +-- ...gedEvent.java => PersonsChangedEvent.java} | 6 +-- ...vedEvent.java => PersonsRemovedEvent.java} | 6 +-- .../{PersonEvent.java => PersonxEvent.java} | 4 +- .../datamodel/AutopsyTreeChildFactory.java | 23 +++++++--- .../sleuthkit/autopsy/datamodel/HostNode.java | 28 ++++++++++-- .../autopsy/datamodel/PersonGroupingNode.java | 43 ++++++++++++++++--- 12 files changed, 125 insertions(+), 53 deletions(-) rename Core/src/org/sleuthkit/autopsy/casemodule/events/{HostAddedEvent.java => HostsAddedEvent.java} (86%) rename Core/src/org/sleuthkit/autopsy/casemodule/events/{HostChangedEvent.java => HostsChangedEvent.java} (85%) rename Core/src/org/sleuthkit/autopsy/casemodule/events/{HostEvent.java => HostsEvent.java} (95%) rename Core/src/org/sleuthkit/autopsy/casemodule/events/{HostRemovedEvent.java => HostsRemovedEvent.java} (85%) rename Core/src/org/sleuthkit/autopsy/casemodule/events/{PersonAddedEvent.java => PersonsAddedEvent.java} (85%) rename Core/src/org/sleuthkit/autopsy/casemodule/events/{PersonChangedEvent.java => PersonsChangedEvent.java} (85%) rename Core/src/org/sleuthkit/autopsy/casemodule/events/{PersonRemovedEvent.java => PersonsRemovedEvent.java} (85%) rename Core/src/org/sleuthkit/autopsy/casemodule/events/{PersonEvent.java => PersonxEvent.java} (94%) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index eb37895a30..ffd226d6d3 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -82,11 +82,14 @@ import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; import org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent; import org.sleuthkit.autopsy.casemodule.events.DataSourceDeletedEvent; import org.sleuthkit.autopsy.casemodule.events.DataSourceNameChangedEvent; -import org.sleuthkit.autopsy.casemodule.events.HostAddedEvent; -import org.sleuthkit.autopsy.casemodule.events.HostChangedEvent; -import org.sleuthkit.autopsy.casemodule.events.HostRemovedEvent; +import org.sleuthkit.autopsy.casemodule.events.HostsAddedEvent; +import org.sleuthkit.autopsy.casemodule.events.HostsChangedEvent; +import org.sleuthkit.autopsy.casemodule.events.HostsRemovedEvent; import org.sleuthkit.autopsy.casemodule.events.OsAccountAddedEvent; import org.sleuthkit.autopsy.casemodule.events.OsAccountChangedEvent; +import org.sleuthkit.autopsy.casemodule.events.PersonsAddedEvent; +import org.sleuthkit.autopsy.casemodule.events.PersonsChangedEvent; +import org.sleuthkit.autopsy.casemodule.events.PersonsRemovedEvent; import org.sleuthkit.autopsy.casemodule.events.ReportAddedEvent; import org.sleuthkit.autopsy.casemodule.multiusercases.CaseNodeData.CaseNodeDataException; import org.sleuthkit.autopsy.casemodule.multiusercases.CoordinationServiceUtils; @@ -135,11 +138,18 @@ import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.FileSystem; import org.sleuthkit.datamodel.Host; +import org.sleuthkit.datamodel.HostManager.HostsCreationEvent; +import org.sleuthkit.datamodel.HostManager.HostsUpdateEvent; +import org.sleuthkit.datamodel.HostManager.HostsDeletionEvent; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.OsAccount; import org.sleuthkit.datamodel.OsAccountManager; import org.sleuthkit.datamodel.OsAccountManager.OsAccountsCreationEvent; import org.sleuthkit.datamodel.OsAccountManager.OsAccountsUpdateEvent; +import org.sleuthkit.datamodel.Person; +import org.sleuthkit.datamodel.PersonManager.PersonsCreationEvent; +import org.sleuthkit.datamodel.PersonManager.PersonsUpdateEvent; +import org.sleuthkit.datamodel.PersonManager.PersonsDeletionEvent; import org.sleuthkit.datamodel.Report; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TimelineManager; @@ -517,7 +527,7 @@ public class Case { * @param event The sleuthkit event for the creation of hosts. */ @Subscribe - public void publishHostsAddedEvent(HostCreationEvent event) { + public void publishHostsAddedEvent(HostsCreationEvent event) { eventPublisher.publish(new HostsAddedEvent( event == null ? Collections.emptyList() : event.getHosts())); } @@ -529,7 +539,7 @@ public class Case { * @param event The sleuthkit event for the updating of hosts. */ @Subscribe - public void publishHostsChangedEvent(HostUpdateEvent event) { + public void publishHostsChangedEvent(HostsUpdateEvent event) { eventPublisher.publish(new HostsChangedEvent( event == null ? Collections.emptyList() : event.getHosts())); } @@ -541,7 +551,7 @@ public class Case { * @param event The sleuthkit event for the deleting of hosts. */ @Subscribe - public void publishHostsDeletedEvent(HostDeletedEvent event) { + public void publishHostsDeletedEvent(HostsDeletionEvent event) { eventPublisher.publish(new HostsRemovedEvent( event == null ? Collections.emptyList() : event.getHosts())); } @@ -553,7 +563,7 @@ public class Case { * @param event The sleuthkit event for the creation of persons. */ @Subscribe - public void publishPersonsAddedEvent(PersonCreationEvent event) { + public void publishPersonsAddedEvent(PersonsCreationEvent event) { eventPublisher.publish(new PersonsAddedEvent( event == null ? Collections.emptyList() : event.getPersons())); } @@ -565,7 +575,7 @@ public class Case { * @param event The sleuthkit event for the updating of persons. */ @Subscribe - public void publishPersonsChangedEvent(PersonUpdateEvent event) { + public void publishPersonsChangedEvent(PersonsUpdateEvent event) { eventPublisher.publish(new PersonsChangedEvent( event == null ? Collections.emptyList() : event.getPersons())); } @@ -577,7 +587,7 @@ public class Case { * @param event The sleuthkit event for the deleting of persons. */ @Subscribe - public void publishPersonsDeletedEvent(PersonDeletedEvent event) { + public void publishPersonsDeletedEvent(PersonsDeletionEvent event) { eventPublisher.publish(new PersonsRemovedEvent( event == null ? Collections.emptyList() : event.getPersons())); } @@ -1793,7 +1803,7 @@ public class Case { * @param host The host that has been added. */ public void notifyHostAdded(Host host) { - eventPublisher.publish(new HostAddedEvent(Collections.singletonList(host))); + eventPublisher.publish(new HostsAddedEvent(Collections.singletonList(host))); } /** @@ -1801,7 +1811,7 @@ public class Case { * @param newValue The host that has been updated. */ public void notifyHostChanged(Host newValue) { - eventPublisher.publish(new HostChangedEvent(Collections.singletonList(newValue))); + eventPublisher.publish(new HostsChangedEvent(Collections.singletonList(newValue))); } /** @@ -1809,7 +1819,7 @@ public class Case { * @param host The host that has been deleted. */ public void notifyHostDeleted(Host host) { - eventPublisher.publish(new HostRemovedEvent(Collections.singletonList(host))); + eventPublisher.publish(new HostsRemovedEvent(Collections.singletonList(host))); } /** @@ -1817,7 +1827,7 @@ public class Case { * @param person The person that has been added. */ public void notifyPersonAdded(Person person) { - eventPublisher.publish(new PersonAddedEvent(Collections.singletonList(person))); + eventPublisher.publish(new PersonsAddedEvent(Collections.singletonList(person))); } /** @@ -1825,7 +1835,7 @@ public class Case { * @param newValue The person that has been updated. */ public void notifyPersonChanged(Person newValue) { - eventPublisher.publish(new PersonChangedEvent(Collections.singletonList(newValue))); + eventPublisher.publish(new PersonsChangedEvent(Collections.singletonList(newValue))); } /** @@ -1833,7 +1843,7 @@ public class Case { * @param person The person that has been deleted. */ public void notifyPersonDeleted(Person person) { - eventPublisher.publish(new PersonRemovedEvent(Collections.singletonList(person))); + eventPublisher.publish(new PersonsRemovedEvent(Collections.singletonList(person))); } /** diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostAddedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsAddedEvent.java similarity index 86% rename from Core/src/org/sleuthkit/autopsy/casemodule/events/HostAddedEvent.java rename to Core/src/org/sleuthkit/autopsy/casemodule/events/HostsAddedEvent.java index 3676c475c0..c10b66face 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostAddedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsAddedEvent.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Host; /** * Event fired when new hosts are added. */ -public class HostAddedEvent extends HostEvent { +public class HostsAddedEvent extends HostsEvent { private static final long serialVersionUID = 1L; @@ -33,7 +33,7 @@ public class HostAddedEvent extends HostEvent { * Main constructor. * @param dataModelObjects The hosts that have been added. */ - public HostAddedEvent(List dataModelObjects) { - super(Case.Events.HOST_ADDED.name(), dataModelObjects); + public HostsAddedEvent(List dataModelObjects) { + super(Case.Events.HOSTS_ADDED.name(), dataModelObjects); } } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostChangedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsChangedEvent.java similarity index 85% rename from Core/src/org/sleuthkit/autopsy/casemodule/events/HostChangedEvent.java rename to Core/src/org/sleuthkit/autopsy/casemodule/events/HostsChangedEvent.java index 37e589ea61..a5b8692c03 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostChangedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsChangedEvent.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Host; /** * Event fired when hosts are changed. */ -public class HostChangedEvent extends HostEvent { +public class HostsChangedEvent extends HostsEvent { private static final long serialVersionUID = 1L; @@ -35,7 +35,7 @@ public class HostChangedEvent extends HostEvent { * @param dataModelObjects The new values for the hosts that have been * changed. */ - public HostChangedEvent(List dataModelObjects) { - super(Case.Events.HOST_CHANGED.name(), dataModelObjects); + public HostsChangedEvent(List dataModelObjects) { + super(Case.Events.HOSTS_CHANGED.name(), dataModelObjects); } } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java similarity index 95% rename from Core/src/org/sleuthkit/autopsy/casemodule/events/HostEvent.java rename to Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java index 0f2e1ed0e4..1ba225f4a2 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java @@ -31,7 +31,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Base event class for when something pertaining to hosts changes. */ -public class HostEvent extends TskDataModelChangeEvent { +public class HostsEvent extends TskDataModelChangeEvent { /** * Retrieves a list of ids from a list of hosts. @@ -62,7 +62,7 @@ public class HostEvent extends TskDataModelChangeEvent { * type. * @param dataModelObjects The list of hosts for the event. */ - protected HostEvent(String eventName, List dataModelObjects) { + protected HostsEvent(String eventName, List dataModelObjects) { super(eventName, getIds(dataModelObjects), new ArrayList<>(getSafeList(dataModelObjects))); } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostRemovedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsRemovedEvent.java similarity index 85% rename from Core/src/org/sleuthkit/autopsy/casemodule/events/HostRemovedEvent.java rename to Core/src/org/sleuthkit/autopsy/casemodule/events/HostsRemovedEvent.java index 22cd237629..407b83c32a 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostRemovedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsRemovedEvent.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Host; /** * Event fired when hosts are removed. */ -public class HostRemovedEvent extends HostEvent { +public class HostsRemovedEvent extends HostsEvent { private static final long serialVersionUID = 1L; @@ -33,7 +33,7 @@ public class HostRemovedEvent extends HostEvent { * Main constructor. * @param dataModelObjects The list of hosts that have been deleted. */ - public HostRemovedEvent(List dataModelObjects) { - super(Case.Events.HOST_DELETED.name(), dataModelObjects); + public HostsRemovedEvent(List dataModelObjects) { + super(Case.Events.HOSTS_DELETED.name(), dataModelObjects); } } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonAddedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsAddedEvent.java similarity index 85% rename from Core/src/org/sleuthkit/autopsy/casemodule/events/PersonAddedEvent.java rename to Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsAddedEvent.java index 2d33315f95..0b7a0f6a12 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonAddedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsAddedEvent.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Person; /** * Event fired when new persons are added. */ -public class PersonAddedEvent extends PersonEvent { +public class PersonsAddedEvent extends PersonxEvent { private static final long serialVersionUID = 1L; @@ -33,7 +33,7 @@ public class PersonAddedEvent extends PersonEvent { * Main constructor. * @param dataModelObjects The persons that have been added. */ - public PersonAddedEvent(List dataModelObjects) { - super(Case.Events.PERSON_ADDED.name(), dataModelObjects); + public PersonsAddedEvent(List dataModelObjects) { + super(Case.Events.PERSONS_ADDED.name(), dataModelObjects); } } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonChangedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsChangedEvent.java similarity index 85% rename from Core/src/org/sleuthkit/autopsy/casemodule/events/PersonChangedEvent.java rename to Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsChangedEvent.java index 9209497de8..f3027cf22b 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonChangedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsChangedEvent.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Person; /** * Event fired when persons are changed. */ -public class PersonChangedEvent extends PersonEvent { +public class PersonsChangedEvent extends PersonxEvent { private static final long serialVersionUID = 1L; @@ -35,7 +35,7 @@ public class PersonChangedEvent extends PersonEvent { * @param dataModelObjects The new values for the persons that have been * changed. */ - public PersonChangedEvent(List dataModelObjects) { - super(Case.Events.PERSON_CHANGED.name(), dataModelObjects); + public PersonsChangedEvent(List dataModelObjects) { + super(Case.Events.PERSONS_CHANGED.name(), dataModelObjects); } } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonRemovedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsRemovedEvent.java similarity index 85% rename from Core/src/org/sleuthkit/autopsy/casemodule/events/PersonRemovedEvent.java rename to Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsRemovedEvent.java index a33af32cc7..4a5c717002 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonRemovedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsRemovedEvent.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Person; /** * Event fired when persons are removed. */ -public class PersonRemovedEvent extends PersonEvent { +public class PersonsRemovedEvent extends PersonxEvent { private static final long serialVersionUID = 1L; @@ -33,7 +33,7 @@ public class PersonRemovedEvent extends PersonEvent { * Main constructor. * @param dataModelObjects The list of persons that have been deleted. */ - public PersonRemovedEvent(List dataModelObjects) { - super(Case.Events.PERSON_DELETED.name(), dataModelObjects); + public PersonsRemovedEvent(List dataModelObjects) { + super(Case.Events.PERSONS_DELETED.name(), dataModelObjects); } } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonxEvent.java similarity index 94% rename from Core/src/org/sleuthkit/autopsy/casemodule/events/PersonEvent.java rename to Core/src/org/sleuthkit/autopsy/casemodule/events/PersonxEvent.java index f41ae6ff86..6582f7bc83 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonxEvent.java @@ -31,7 +31,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Base event class for when something pertaining to persons changes. */ -public class PersonEvent extends TskDataModelChangeEvent { +public class PersonxEvent extends TskDataModelChangeEvent { /** * Retrieves a list of ids from a list of persons. @@ -62,7 +62,7 @@ public class PersonEvent extends TskDataModelChangeEvent { * type. * @param dataModelObjects The list of persons for the event. */ - protected PersonEvent(String eventName, List dataModelObjects) { + protected PersonxEvent(String eventName, List dataModelObjects) { super(eventName, getIds(dataModelObjects), new ArrayList<>(getSafeList(dataModelObjects))); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java index e7af2149d6..3f02162159 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; import java.util.logging.Level; +import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; import org.openide.nodes.ChildFactory; import org.openide.nodes.Node; @@ -47,6 +48,18 @@ import org.sleuthkit.datamodel.TskCoreException; */ public final class AutopsyTreeChildFactory extends ChildFactory.Detachable { + private static final Set LISTENING_EVENTS = EnumSet.of( + Case.Events.DATA_SOURCE_ADDED, + Case.Events.HOSTS_ADDED, + Case.Events.HOSTS_DELETED, + Case.Events.PERSONS_ADDED, + Case.Events.PERSONS_DELETED + ); + + private static final Set LISTENING_EVENT_NAMES = LISTENING_EVENTS.stream() + .map(evt -> evt.name()) + .collect(Collectors.toSet()); + private static final Logger logger = Logger.getLogger(AutopsyTreeChildFactory.class.getName()); /** @@ -56,7 +69,7 @@ public final class AutopsyTreeChildFactory extends ChildFactory.Detachable h != null && h.getId() == hostId) + .findFirst() + .ifPresent((newHost) -> setDisplayName(newHost.getName())); + } + } + }; + private final Host host; + private final Long hostId; /** * Main constructor for HostDataSources key where data source children @@ -197,7 +215,9 @@ public class HostNode extends DisplayableItemNode { String safeName = (host == null || host.getName() == null) ? Bundle.HostGroupingNode_unknownHostNode_title() : host.getName(); - + + hostId = host == null ? null : host.getId(); + Case.addEventTypeSubscriber(EnumSet.of(Case.Events.HOSTS_CHANGED), hostChangePcl); super.setName(safeName); super.setDisplayName(safeName); this.setIconBaseWithExtension(ICON_PATH); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java index 11b4c2e647..f1bb42665f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java @@ -23,7 +23,10 @@ import java.beans.PropertyChangeListener; import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.logging.Level; +import java.util.stream.Collectors; import javax.swing.Action; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; @@ -33,6 +36,8 @@ import org.openide.util.NbBundle.Messages; import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.casemodule.events.HostsChangedEvent; +import org.sleuthkit.autopsy.casemodule.events.PersonsChangedEvent; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.persons.DeletePersonAction; import org.sleuthkit.autopsy.datamodel.persons.EditPersonAction; @@ -56,6 +61,11 @@ public class PersonGroupingNode extends DisplayableItemNode { private static final Logger logger = Logger.getLogger(PersonChildren.class.getName()); + private static final Set CHILD_EVENTS = EnumSet.of(Case.Events.HOSTS_ADDED, Case.Events.HOSTS_CHANGED); + private static final Set CHILD_EVENTS_STR = CHILD_EVENTS.stream() + .map(ev -> ev.name()) + .collect(Collectors.toSet()); + private final Person person; /** @@ -68,15 +78,13 @@ public class PersonGroupingNode extends DisplayableItemNode { } /** - * Listener for handling DATA_SOURCE_ADDED and DATA_SOURCE_DELETED - * events. + * Listener for handling adding and removing host events. */ - private final PropertyChangeListener pcl = new PropertyChangeListener() { + private final PropertyChangeListener hostAddedDeletedPcl = new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String eventType = evt.getPropertyName(); - if (eventType.equals(Case.Events.DATA_SOURCE_ADDED.toString()) - || eventType.equals(Case.Events.DATA_SOURCE_DELETED.toString())) { + if (eventType != null && CHILD_EVENTS_STR.contains(eventType)) { refresh(true); } } @@ -84,12 +92,12 @@ public class PersonGroupingNode extends DisplayableItemNode { @Override protected void addNotify() { - Case.addEventTypeSubscriber(EnumSet.of(Case.Events.DATA_SOURCE_ADDED), pcl); + Case.addEventTypeSubscriber(CHILD_EVENTS, hostAddedDeletedPcl); } @Override protected void removeNotify() { - Case.removeEventTypeSubscriber(EnumSet.of(Case.Events.DATA_SOURCE_ADDED), pcl); + Case.removeEventTypeSubscriber(CHILD_EVENTS, hostAddedDeletedPcl); } @Override @@ -117,7 +125,24 @@ public class PersonGroupingNode extends DisplayableItemNode { } private final Person person; + private final Long personId; + /** + * Listener for handling person change events. + */ + private final PropertyChangeListener personChangePcl = new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent evt) { + String eventType = evt.getPropertyName(); + if (personId != null && eventType.equals(Case.Events.PERSONS_CHANGED.toString()) && evt instanceof PersonsChangedEvent) { + ((PersonsChangedEvent) evt).getNewValue().stream() + .filter(p -> p != null && p.getId() == personId) + .findFirst() + .ifPresent((newPerson) -> setDisplayName(newPerson.getName())); + } + } + }; + /** * Main constructor. * @@ -134,7 +159,11 @@ public class PersonGroupingNode extends DisplayableItemNode { super.setDisplayName(safeName); this.setIconBaseWithExtension(ICON_PATH); this.person = person; + this.personId = person == null ? null : person.getId(); + Case.addEventTypeSubscriber(EnumSet.of(Case.Events.PERSONS_CHANGED), personChangePcl); } + + @Override public boolean isLeafTypeNode() { From a108ab5594c8b16d15f8ffbba2a52c7454b08608 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 2 Mar 2021 09:18:19 -0500 Subject: [PATCH 22/37] fix --- .../src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java index f1bb42665f..635b4b5aef 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java @@ -61,7 +61,7 @@ public class PersonGroupingNode extends DisplayableItemNode { private static final Logger logger = Logger.getLogger(PersonChildren.class.getName()); - private static final Set CHILD_EVENTS = EnumSet.of(Case.Events.HOSTS_ADDED, Case.Events.HOSTS_CHANGED); + private static final Set CHILD_EVENTS = EnumSet.of(Case.Events.HOSTS_ADDED, Case.Events.HOSTS_DELETED); private static final Set CHILD_EVENTS_STR = CHILD_EVENTS.stream() .map(ev -> ev.name()) .collect(Collectors.toSet()); From 525a6662d8d5649707717bc9e612dfd2f8f740e4 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 2 Mar 2021 13:09:06 -0500 Subject: [PATCH 23/37] weak listeners --- .../datamodel/AutopsyTreeChildFactory.java | 3 +-- .../sleuthkit/autopsy/datamodel/HostNode.java | 11 ++++++++--- .../autopsy/datamodel/PersonGroupingNode.java | 17 +++++++++-------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java index 3f02162159..288626f209 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java @@ -23,7 +23,6 @@ import java.beans.PropertyChangeListener; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -55,7 +54,7 @@ public final class AutopsyTreeChildFactory extends ChildFactory.Detachable LISTENING_EVENT_NAMES = LISTENING_EVENTS.stream() .map(evt -> evt.name()) .collect(Collectors.toSet()); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/HostNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/HostNode.java index 262fb4a903..5e4df77a3f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/HostNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/HostNode.java @@ -33,6 +33,7 @@ import org.openide.nodes.Node; import org.openide.nodes.Sheet; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; +import org.openide.util.WeakListeners; import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -175,7 +176,10 @@ public class HostNode extends DisplayableItemNode { ((HostsChangedEvent) evt).getNewValue().stream() .filter(h -> h != null && h.getId() == hostId) .findFirst() - .ifPresent((newHost) -> setDisplayName(newHost.getName())); + .ifPresent((newHost) -> { + setName(newHost.getName()); + setDisplayName(newHost.getName()); + }); } } }; @@ -215,9 +219,10 @@ public class HostNode extends DisplayableItemNode { String safeName = (host == null || host.getName() == null) ? Bundle.HostGroupingNode_unknownHostNode_title() : host.getName(); - + hostId = host == null ? null : host.getId(); - Case.addEventTypeSubscriber(EnumSet.of(Case.Events.HOSTS_CHANGED), hostChangePcl); + Case.addEventTypeSubscriber(EnumSet.of(Case.Events.HOSTS_CHANGED), + WeakListeners.propertyChange(hostChangePcl, this)); super.setName(safeName); super.setDisplayName(safeName); this.setIconBaseWithExtension(ICON_PATH); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java index 635b4b5aef..5cc921b269 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java @@ -23,7 +23,6 @@ import java.beans.PropertyChangeListener; import java.util.Collections; import java.util.EnumSet; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; @@ -33,10 +32,10 @@ import org.openide.nodes.Children; import org.openide.nodes.Sheet; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; +import org.openide.util.WeakListeners; import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.casemodule.events.HostsChangedEvent; import org.sleuthkit.autopsy.casemodule.events.PersonsChangedEvent; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.persons.DeletePersonAction; @@ -65,7 +64,7 @@ public class PersonGroupingNode extends DisplayableItemNode { private static final Set CHILD_EVENTS_STR = CHILD_EVENTS.stream() .map(ev -> ev.name()) .collect(Collectors.toSet()); - + private final Person person; /** @@ -138,11 +137,14 @@ public class PersonGroupingNode extends DisplayableItemNode { ((PersonsChangedEvent) evt).getNewValue().stream() .filter(p -> p != null && p.getId() == personId) .findFirst() - .ifPresent((newPerson) -> setDisplayName(newPerson.getName())); + .ifPresent((newPerson) -> { + setName(newPerson.getName()); + setDisplayName(newPerson.getName()); + }); } } }; - + /** * Main constructor. * @@ -160,10 +162,9 @@ public class PersonGroupingNode extends DisplayableItemNode { this.setIconBaseWithExtension(ICON_PATH); this.person = person; this.personId = person == null ? null : person.getId(); - Case.addEventTypeSubscriber(EnumSet.of(Case.Events.PERSONS_CHANGED), personChangePcl); + Case.addEventTypeSubscriber(EnumSet.of(Case.Events.PERSONS_CHANGED), + WeakListeners.propertyChange(personChangePcl, this)); } - - @Override public boolean isLeafTypeNode() { From 3d8933831fdf9592ac6e80ea9e522fd7880f0b11 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 2 Mar 2021 13:48:38 -0500 Subject: [PATCH 24/37] updates --- .../org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java | 6 +++++- .../modules/portablecase/PortableCaseReportModule.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java index 5cc921b269..cb50ced29f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java @@ -60,7 +60,11 @@ public class PersonGroupingNode extends DisplayableItemNode { private static final Logger logger = Logger.getLogger(PersonChildren.class.getName()); - private static final Set CHILD_EVENTS = EnumSet.of(Case.Events.HOSTS_ADDED, Case.Events.HOSTS_DELETED); + private static final Set CHILD_EVENTS = EnumSet.of( + Case.Events.HOSTS_ADDED, + Case.Events.HOSTS_DELETED, + Case.Events.PERSONS_CHANGED); + private static final Set CHILD_EVENTS_STR = CHILD_EVENTS.stream() .map(ev -> ev.name()) .collect(Collectors.toSet()); diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java index cc2adf377a..56bb8e9133 100644 --- a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java @@ -1085,7 +1085,7 @@ public class PortableCaseReportModule implements ReportModule { Host newHost = null; if (content instanceof DataSource) { Host oldHost = ((DataSource)content).getHost(); - newHost = portableSkCase.getHostManager().getOrCreateHost(oldHost.getName()); + newHost = portableSkCase.getHostManager().createHost(oldHost.getName()); } CaseDbTransaction trans = portableSkCase.beginTransaction(); From 81c776aff10b0921f96fd3549f0c6355b473d9c1 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 2 Mar 2021 14:59:05 -0500 Subject: [PATCH 25/37] class rename --- .../autopsy/casemodule/events/PersonsAddedEvent.java | 2 +- .../autopsy/casemodule/events/PersonsChangedEvent.java | 2 +- .../events/{PersonxEvent.java => PersonsEvent.java} | 4 ++-- .../autopsy/casemodule/events/PersonsRemovedEvent.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename Core/src/org/sleuthkit/autopsy/casemodule/events/{PersonxEvent.java => PersonsEvent.java} (95%) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsAddedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsAddedEvent.java index 0b7a0f6a12..e2a8a7aabd 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsAddedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsAddedEvent.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Person; /** * Event fired when new persons are added. */ -public class PersonsAddedEvent extends PersonxEvent { +public class PersonsAddedEvent extends PersonsEvent { private static final long serialVersionUID = 1L; diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsChangedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsChangedEvent.java index f3027cf22b..f375a125bb 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsChangedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsChangedEvent.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Person; /** * Event fired when persons are changed. */ -public class PersonsChangedEvent extends PersonxEvent { +public class PersonsChangedEvent extends PersonsEvent { private static final long serialVersionUID = 1L; diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonxEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsEvent.java similarity index 95% rename from Core/src/org/sleuthkit/autopsy/casemodule/events/PersonxEvent.java rename to Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsEvent.java index 6582f7bc83..ef74585b7f 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonxEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsEvent.java @@ -31,7 +31,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Base event class for when something pertaining to persons changes. */ -public class PersonxEvent extends TskDataModelChangeEvent { +public class PersonsEvent extends TskDataModelChangeEvent { /** * Retrieves a list of ids from a list of persons. @@ -62,7 +62,7 @@ public class PersonxEvent extends TskDataModelChangeEvent { * type. * @param dataModelObjects The list of persons for the event. */ - protected PersonxEvent(String eventName, List dataModelObjects) { + protected PersonsEvent(String eventName, List dataModelObjects) { super(eventName, getIds(dataModelObjects), new ArrayList<>(getSafeList(dataModelObjects))); } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsRemovedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsRemovedEvent.java index 4a5c717002..522c841461 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsRemovedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsRemovedEvent.java @@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Person; /** * Event fired when persons are removed. */ -public class PersonsRemovedEvent extends PersonxEvent { +public class PersonsRemovedEvent extends PersonsEvent { private static final long serialVersionUID = 1L; From 14b58936a0a751101c5ac792433115f9ca7efc0d Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 4 Mar 2021 07:29:45 -0500 Subject: [PATCH 26/37] update to include event --- .../sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java index 288626f209..c909ddaad3 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java @@ -52,7 +52,8 @@ public final class AutopsyTreeChildFactory extends ChildFactory.Detachable LISTENING_EVENT_NAMES = LISTENING_EVENTS.stream() From 5b5fed78eb7bd6996da14541ab9ffd912b4ecd73 Mon Sep 17 00:00:00 2001 From: apriestman Date: Thu, 4 Mar 2021 10:58:27 -0500 Subject: [PATCH 27/37] Remove unused import --- .../org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java index 856b6eec96..7205d2d89c 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DataSourcesByTypeNode.java @@ -33,7 +33,6 @@ import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.Host.HostStatus; import org.sleuthkit.datamodel.TskCoreException; /** From 86820e98eecc711f2bfa27b96c79b6c026964c44 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 4 Mar 2021 14:20:52 -0500 Subject: [PATCH 28/37] 7370-Substitute SID for OS account Object ID everywhere SID is not in output already --- test/script/tskdbdiff.py | 66 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index 0d4e7f7ab9..d70a77adc3 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -329,7 +329,8 @@ class TskDbDiff(object): id_legacy_artifact_types = build_id_legacy_artifact_types_table(conn.cursor(), isMultiUser) id_reports_table = build_id_reports_table(conn.cursor(), isMultiUser) id_images_table = build_id_image_names_table(conn.cursor(), isMultiUser) - id_obj_path_table = build_id_obj_path_table(id_files_table, id_objects_table, id_artifact_types_table, id_reports_table, id_images_table) + id_accounts_table = build_id_accounts_table(conn.cursor(), isMultiUser) + id_obj_path_table = build_id_obj_path_table(id_files_table, id_objects_table, id_artifact_types_table, id_reports_table, id_images_table, id_accounts_table) if isMultiUser: # Use PostgreSQL os.environ['PGPASSWORD']=pgSettings.password @@ -352,7 +353,7 @@ class TskDbDiff(object): if 'INSERT INTO image_gallery_groups_seen' in dump_line: dump_line = '' continue; - dump_line = normalize_db_entry(dump_line, id_obj_path_table, id_vs_parts_table, id_vs_info_table, id_fs_info_table, id_objects_table, id_reports_table, id_images_table, id_legacy_artifact_types) + dump_line = normalize_db_entry(dump_line, id_obj_path_table, id_vs_parts_table, id_vs_info_table, id_fs_info_table, id_objects_table, id_reports_table, id_images_table, id_legacy_artifact_types, id_accounts_table) db_log.write('%s\n' % dump_line) dump_line = '' postgreSQL_db.close() @@ -366,7 +367,7 @@ class TskDbDiff(object): for line in conn.iterdump(): if 'INSERT INTO "image_gallery_groups_seen"' in line: continue - line = normalize_db_entry(line, id_obj_path_table, id_vs_parts_table, id_vs_info_table, id_fs_info_table, id_objects_table, id_reports_table, id_images_table, id_legacy_artifact_types) + line = normalize_db_entry(line, id_obj_path_table, id_vs_parts_table, id_vs_info_table, id_fs_info_table, id_objects_table, id_reports_table, id_images_table, id_legacy_artifact_types, id_accounts_table) db_log.write('%s\n' % line) # Now sort the file srtcmdlst = ["sort", dump_file, "-o", dump_file] @@ -419,7 +420,7 @@ class PGSettings(object): return self.password -def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info_table, objects_table, reports_table, images_table, artifact_table): +def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info_table, objects_table, reports_table, images_table, artifact_table, accounts_table): """ Make testing more consistent and reasonable by doctoring certain db entries. Args: @@ -442,6 +443,7 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info ig_groups_index = line.find('INSERT INTO "image_gallery_groups"') > -1 or line.find('INSERT INTO image_gallery_groups ') > -1 ig_groups_seen_index = line.find('INSERT INTO "image_gallery_groups_seen"') > -1 or line.find('INSERT INTO image_gallery_groups_seen ') > -1 os_account_index = line.find('INSERT INTO "tsk_os_accounts"') > -1 or line.find('INSERT INTO tsk_os_accounts') > -1 + os_accaount_attr_index = line.find('INSERT INTO "tsk_os_account_attributes"') > -1 or line.find('INSERT INTO tsk_os_account_attributes') > -1 parens = line[line.find('(') + 1 : line.rfind(')')] no_space_parens = parens.replace(" ", "") @@ -569,6 +571,8 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info parent_path = fs_info_table[parent_id] elif parent_id in images_table.keys(): parent_path = images_table[parent_id] + elif parent_id in accounts_table.keys(): + parent_path = accounts_table[parent_id] elif parent_id == 'NULL': parent_path = "NULL" @@ -636,8 +640,42 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info newLine = ('INSERT INTO "tsk_event_descriptions" VALUES(' + ','.join(fields_list[1:]) + ');') # remove report_id return newLine elif os_account_index: - newLine = ('INSERT INTO "tsk_os_accounts" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id + newLine = ('INSERT INTO "tsk_os_accounts" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id since value that would be substituted is in diff line already return newLine + elif os_accaount_attr_index: + #substitue the account object id for a non changing value + os_account_id = fields_list[1] + if os_account_id in files_table.keys(): + fields_list[1] = files_table[parent_id] + elif os_account_id in vs_parts_table.keys(): + fields_list[1] = vs_parts_table[parent_id] + elif os_account_id in vs_info_table.keys(): + fields_list[1] = vs_info_table[parent_id] + elif os_account_id in fs_info_table.keys(): + fields_list[1] = fs_info_table[parent_id] + elif os_account_id in images_table.keys(): + fields_list[1] = images_table[parent_id] + elif os_account_id in accounts_table.keys(): + fields_list[1] = accounts_table[parent_id] + elif os_account_id == 'NULL': + fields_list[1] = "NULL" + #substitue the source object id for a non changing value + source_obj_id = fields_list[3] + if os_account_id in files_table.keys(): + fields_list[1] = files_table[parent_id] + elif os_account_id in vs_parts_table.keys(): + fields_list[1] = vs_parts_table[parent_id] + elif os_account_id in vs_info_table.keys(): + fields_list[1] = vs_info_table[parent_id] + elif os_account_id in fs_info_table.keys(): + fields_list[1] = fs_info_table[parent_id] + elif os_account_id in images_table.keys(): + fields_list[1] = images_table[parent_id] + elif os_account_id in accounts_table.keys(): + fields_list[1] = accounts_table[parent_id] + elif os_account_id == 'NULL': + fields_list[1] = "NULL" + newLine = = ('INSERT INTO "tsk_os_account_attributes" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id else: return line @@ -758,8 +796,18 @@ def build_id_reports_table(db_cursor, isPostgreSQL): mapping = dict([(row[0], row[1]) for row in sql_select_execute(db_cursor, isPostgreSQL, "SELECT obj_id, path FROM reports")]) return mapping +def build_id_accounts_table(db_cursor, isPostgreSQL): + """Build the map of object ids to OS account SIDs. -def build_id_obj_path_table(files_table, objects_table, artifacts_table, reports_table, images_table): + Args: + db_cursor: the database cursor + """ + # for each row in the db, take the object id and account SID then creates a tuple in the dictionary + # with the object id as the key and the OS Account's SID as the value + mapping = dict([(row[0], row[1]) for row in sql_select_execute(db_cursor, isPostgreSQL, "SELECT os_account_obj_id, unique_id FROM tsk_os_accounts")]) + return mapping + +def build_id_obj_path_table(files_table, objects_table, artifacts_table, reports_table, images_table, accounts_table): """Build the map of object ids to artifact ids. Args: @@ -767,6 +815,8 @@ def build_id_obj_path_table(files_table, objects_table, artifacts_table, reports objects_table: obj_id, par_obj_id, type artifacts_table: obj_id, artifact_type_name reports_table: obj_id, path + images_table: obj_id, name + accounts_table: obj_id, unique_id """ # make a copy of files_table and update it with new data from artifacts_table and reports_table mapping = files_table.copy() @@ -786,6 +836,10 @@ def build_id_obj_path_table(files_table, objects_table, artifacts_table, reports elif par_obj_id in images_table.keys(): path = images_table[par_obj_id] mapping[k] = path + "/" + artifacts_table[k] + elif k in accounts_table.keys(): # For an OS Account object ID we use its unique_id field which is the account SID + unique_id = v[0] + if unique_id is not None: + mapping[k] = accounts_table[k] elif v[0] not in mapping.keys(): if v[0] in artifacts_table.keys(): par_obj_id = objects_table[v[0]] From 7c157dad81ce38257d19a4c60dedb584a91f1ff8 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 4 Mar 2021 14:23:12 -0500 Subject: [PATCH 29/37] 7370 fix typo --- test/script/tskdbdiff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index d70a77adc3..e0081e06b9 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -443,7 +443,7 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info ig_groups_index = line.find('INSERT INTO "image_gallery_groups"') > -1 or line.find('INSERT INTO image_gallery_groups ') > -1 ig_groups_seen_index = line.find('INSERT INTO "image_gallery_groups_seen"') > -1 or line.find('INSERT INTO image_gallery_groups_seen ') > -1 os_account_index = line.find('INSERT INTO "tsk_os_accounts"') > -1 or line.find('INSERT INTO tsk_os_accounts') > -1 - os_accaount_attr_index = line.find('INSERT INTO "tsk_os_account_attributes"') > -1 or line.find('INSERT INTO tsk_os_account_attributes') > -1 + os_account_attr_index = line.find('INSERT INTO "tsk_os_account_attributes"') > -1 or line.find('INSERT INTO tsk_os_account_attributes') > -1 parens = line[line.find('(') + 1 : line.rfind(')')] no_space_parens = parens.replace(" ", "") @@ -642,7 +642,7 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info elif os_account_index: newLine = ('INSERT INTO "tsk_os_accounts" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id since value that would be substituted is in diff line already return newLine - elif os_accaount_attr_index: + elif os_account_attr_index: #substitue the account object id for a non changing value os_account_id = fields_list[1] if os_account_id in files_table.keys(): From 3a663c57b04495eac228110bacc416b58c796d5a Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 4 Mar 2021 14:24:32 -0500 Subject: [PATCH 30/37] 7370 fix extra = in assignment --- test/script/tskdbdiff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index e0081e06b9..f375934783 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -675,7 +675,7 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info fields_list[1] = accounts_table[parent_id] elif os_account_id == 'NULL': fields_list[1] = "NULL" - newLine = = ('INSERT INTO "tsk_os_account_attributes" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id + newLine = ('INSERT INTO "tsk_os_account_attributes" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id else: return line From fb2dbaa6db889c66b6ac257c290590214b585c13 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 4 Mar 2021 14:31:06 -0500 Subject: [PATCH 31/37] 7370 fix copy paste that would have left source object id unchanged --- test/script/tskdbdiff.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index f375934783..a50911b05c 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -661,20 +661,20 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info fields_list[1] = "NULL" #substitue the source object id for a non changing value source_obj_id = fields_list[3] - if os_account_id in files_table.keys(): - fields_list[1] = files_table[parent_id] - elif os_account_id in vs_parts_table.keys(): - fields_list[1] = vs_parts_table[parent_id] - elif os_account_id in vs_info_table.keys(): - fields_list[1] = vs_info_table[parent_id] - elif os_account_id in fs_info_table.keys(): - fields_list[1] = fs_info_table[parent_id] - elif os_account_id in images_table.keys(): - fields_list[1] = images_table[parent_id] - elif os_account_id in accounts_table.keys(): - fields_list[1] = accounts_table[parent_id] - elif os_account_id == 'NULL': - fields_list[1] = "NULL" + if source_obj_id in files_table.keys(): + fields_list[3] = files_table[parent_id] + elif source_obj_id in vs_parts_table.keys(): + fields_list[3] = vs_parts_table[parent_id] + elif source_obj_id in vs_info_table.keys(): + fields_list[3] = vs_info_table[parent_id] + elif source_obj_id in fs_info_table.keys(): + fields_list[3] = fs_info_table[parent_id] + elif source_obj_id in images_table.keys(): + fields_list[3] = images_table[parent_id] + elif source_obj_id in accounts_table.keys(): + fields_list[3] = accounts_table[parent_id] + elif source_obj_id == 'NULL': + fields_list[3] = "NULL" newLine = ('INSERT INTO "tsk_os_account_attributes" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id else: return line From 9106868299f83c3f9b5b04fae9c56e2f57d3c780 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 4 Mar 2021 14:33:12 -0500 Subject: [PATCH 32/37] 7370 fix the rest of copy paste error --- test/script/tskdbdiff.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index a50911b05c..4c19de1aa9 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -646,33 +646,33 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info #substitue the account object id for a non changing value os_account_id = fields_list[1] if os_account_id in files_table.keys(): - fields_list[1] = files_table[parent_id] + fields_list[1] = files_table[os_account_id] elif os_account_id in vs_parts_table.keys(): - fields_list[1] = vs_parts_table[parent_id] + fields_list[1] = vs_parts_table[os_account_id] elif os_account_id in vs_info_table.keys(): - fields_list[1] = vs_info_table[parent_id] + fields_list[1] = vs_info_table[os_account_id] elif os_account_id in fs_info_table.keys(): - fields_list[1] = fs_info_table[parent_id] + fields_list[1] = fs_info_table[os_account_id] elif os_account_id in images_table.keys(): - fields_list[1] = images_table[parent_id] + fields_list[1] = images_table[os_account_id] elif os_account_id in accounts_table.keys(): - fields_list[1] = accounts_table[parent_id] + fields_list[1] = accounts_table[os_account_id] elif os_account_id == 'NULL': fields_list[1] = "NULL" #substitue the source object id for a non changing value source_obj_id = fields_list[3] if source_obj_id in files_table.keys(): - fields_list[3] = files_table[parent_id] + fields_list[3] = files_table[source_obj_id] elif source_obj_id in vs_parts_table.keys(): - fields_list[3] = vs_parts_table[parent_id] + fields_list[3] = vs_parts_table[source_obj_id] elif source_obj_id in vs_info_table.keys(): - fields_list[3] = vs_info_table[parent_id] + fields_list[3] = vs_info_table[source_obj_id] elif source_obj_id in fs_info_table.keys(): - fields_list[3] = fs_info_table[parent_id] + fields_list[3] = fs_info_table[source_obj_id] elif source_obj_id in images_table.keys(): - fields_list[3] = images_table[parent_id] + fields_list[3] = images_table[source_obj_id] elif source_obj_id in accounts_table.keys(): - fields_list[3] = accounts_table[parent_id] + fields_list[3] = accounts_table[source_obj_id] elif source_obj_id == 'NULL': fields_list[3] = "NULL" newLine = ('INSERT INTO "tsk_os_account_attributes" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id From fa18fc5b4f9604dc02758b56101a885f35509dea Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 4 Mar 2021 16:46:18 -0500 Subject: [PATCH 33/37] 7370 add missing return statement --- test/script/tskdbdiff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index 4c19de1aa9..dc7e5ee691 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -676,6 +676,7 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info elif source_obj_id == 'NULL': fields_list[3] = "NULL" newLine = ('INSERT INTO "tsk_os_account_attributes" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id + return newLine else: return line From a545401df86298961c9577995451d742cbbcd64c Mon Sep 17 00:00:00 2001 From: apriestman Date: Fri, 5 Mar 2021 12:30:50 -0500 Subject: [PATCH 34/37] Add "ImageGallery" to name of worker thread --- .../sleuthkit/autopsy/imagegallery/ImageGalleryController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 2f13a48fd2..9513163587 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -659,7 +659,7 @@ public final class ImageGalleryController { private static ListeningExecutorService getNewDBExecutor() { return MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor( - new ThreadFactoryBuilder().setNameFormat("DB-Worker-Thread-%d").build())); + new ThreadFactoryBuilder().setNameFormat("ImageGallery-DB-Worker-Thread-%d").build())); } /** From 7d2244b84934fcfd843aefaa33e995881e0f49a7 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 5 Mar 2021 15:13:33 -0500 Subject: [PATCH 35/37] fix view context unknown persons issue --- .../autopsy/datamodel/PersonGroupingNode.java | 10 +++++++++- .../directorytree/DirectoryTreeTopComponent.java | 7 +++---- .../autopsy/directorytree/ViewContextAction.java | 4 +++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java index 1098ad7322..569d93eb3e 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGroupingNode.java @@ -52,6 +52,14 @@ import org.sleuthkit.datamodel.TskCoreException; public class PersonGroupingNode extends DisplayableItemNode { private static final String ICON_PATH = "org/sleuthkit/autopsy/images/person.png"; + + /** + * Returns the id of an unknown persons node. This can be used with a node lookup. + * @return The id of an unknown persons node. + */ + public static String getUnknownPersonId() { + return Bundle.PersonNode_unknownPersonNode_title(); + } /** * Responsible for creating the host children of this person. @@ -157,7 +165,7 @@ public class PersonGroupingNode extends DisplayableItemNode { */ private static String getDisplayName(Person person) { return (person == null || person.getName() == null) - ? Bundle.PersonNode_unknownPersonNode_title() + ? getUnknownPersonId() : person.getName(); } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 4a3a432aed..3222173ba3 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -35,7 +35,6 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.prefs.PreferenceChangeEvent; import java.util.prefs.PreferenceChangeListener; -import java.util.stream.Stream; import javax.swing.Action; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; @@ -82,8 +81,7 @@ import org.sleuthkit.autopsy.datamodel.InterestingHits; import org.sleuthkit.autopsy.datamodel.KeywordHits; import org.sleuthkit.autopsy.datamodel.ResultsNode; import org.sleuthkit.autopsy.datamodel.AutopsyTreeChildFactory; -import org.sleuthkit.autopsy.datamodel.DataSourceGrouping; -import org.sleuthkit.autopsy.datamodel.DataSourcesByTypeNode; +import org.sleuthkit.autopsy.datamodel.PersonGroupingNode; import org.sleuthkit.autopsy.datamodel.Tags; import org.sleuthkit.autopsy.datamodel.ViewsNode; import org.sleuthkit.autopsy.datamodel.accounts.Accounts; @@ -1086,7 +1084,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat if (node == null) { return null; } else if (node.getLookup().lookup(Host.class) != null - || node.getLookup().lookup(Person.class) != null) { + || node.getLookup().lookup(Person.class) != null + || PersonGroupingNode.getUnknownPersonId().equals(node.getLookup().lookup(String.class))) { Children children = node.getChildren(); Node[] childNodes = children == null ? null : children.getNodes(); if (childNodes != null) { diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java index bb8fd7c561..ca9aa43198 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ViewContextAction.java @@ -49,6 +49,7 @@ import org.sleuthkit.autopsy.datamodel.ContentNodeSelectionInfo; import org.sleuthkit.autopsy.datamodel.DataSourcesByTypeNode; import org.sleuthkit.autopsy.datamodel.DataSourcesNode; import org.sleuthkit.autopsy.datamodel.DisplayableItemNode; +import org.sleuthkit.autopsy.datamodel.PersonGroupingNode; import org.sleuthkit.autopsy.datamodel.RootContentChildren; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -319,7 +320,8 @@ public class ViewContextAction extends AbstractAction { return Collections.emptyList(); } else if (node.getLookup().lookup(Host.class) != null || node.getLookup().lookup(Person.class) != null || - DataSourcesByTypeNode.getNameIdentifier().equals(node.getLookup().lookup(String.class))) { + DataSourcesByTypeNode.getNameIdentifier().equals(node.getLookup().lookup(String.class)) || + PersonGroupingNode.getUnknownPersonId().equals(node.getLookup().lookup(String.class))) { Children children = node.getChildren(); Node[] childNodes = children == null ? null : children.getNodes(); if (childNodes == null) { From 58148f4e42924051f1f20f910aa0c4a084d80bd0 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Fri, 5 Mar 2021 15:40:18 -0500 Subject: [PATCH 36/37] 4186: Remove admin column and field from Os account table and class. --- .../sleuthkit/autopsy/recentactivity/ExtractRegistry.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 1b1661c82e..46504c1886 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -2315,8 +2315,9 @@ class ExtractRegistry extends Extract { // "Default Admin User", "Custom Limited Acct" // and "Default Guest Acct" value = userInfo.get(ACCOUNT_TYPE_KEY); - if (value != null && !value.isEmpty()) { - osAccount.setIsAdmin(value.toLowerCase().contains("Admin")); + if (value != null && !value.isEmpty() && value.toLowerCase().contains("admin")) { + attributes.add(createOsAccountAttribute(ATTRIBUTE_TYPE.TSK_IS_ADMIN, + 1, osAccount, host, regFile)); } value = userInfo.get(USER_COMMENT_KEY); From 7810810a7e2621cd33ad95db1b87c2d183f20b32 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 5 Mar 2021 16:01:21 -0500 Subject: [PATCH 37/37] 7370 fixed by adding casting --- test/script/tskdbdiff.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index dc7e5ee691..85087892b1 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -644,23 +644,10 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info return newLine elif os_account_attr_index: #substitue the account object id for a non changing value - os_account_id = fields_list[1] - if os_account_id in files_table.keys(): - fields_list[1] = files_table[os_account_id] - elif os_account_id in vs_parts_table.keys(): - fields_list[1] = vs_parts_table[os_account_id] - elif os_account_id in vs_info_table.keys(): - fields_list[1] = vs_info_table[os_account_id] - elif os_account_id in fs_info_table.keys(): - fields_list[1] = fs_info_table[os_account_id] - elif os_account_id in images_table.keys(): - fields_list[1] = images_table[os_account_id] - elif os_account_id in accounts_table.keys(): - fields_list[1] = accounts_table[os_account_id] - elif os_account_id == 'NULL': - fields_list[1] = "NULL" + os_account_id = int(fields_list[1]) + fields_list[1] = accounts_table[os_account_id] #substitue the source object id for a non changing value - source_obj_id = fields_list[3] + source_obj_id = int(fields_list[3]) if source_obj_id in files_table.keys(): fields_list[3] = files_table[source_obj_id] elif source_obj_id in vs_parts_table.keys(): @@ -838,9 +825,7 @@ def build_id_obj_path_table(files_table, objects_table, artifacts_table, reports path = images_table[par_obj_id] mapping[k] = path + "/" + artifacts_table[k] elif k in accounts_table.keys(): # For an OS Account object ID we use its unique_id field which is the account SID - unique_id = v[0] - if unique_id is not None: - mapping[k] = accounts_table[k] + mapping[k] = accounts_table[k] elif v[0] not in mapping.keys(): if v[0] in artifacts_table.keys(): par_obj_id = objects_table[v[0]]