From a2e59ad5e12ad84533efb9402ebfa603ae126a72 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 8 Apr 2014 16:30:57 -0400 Subject: [PATCH 01/17] IngestModuleAdapter now has a static moduleRefCount and static methods to access it. --- .../autopsy/ingest/IngestModuleAdapter.java | 19 +++++++++++++++ .../hashdatabase/HashDbIngestModule.java | 24 +++---------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java index 93379de7d4..1c9d1f0aa8 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java @@ -18,11 +18,30 @@ */ package org.sleuthkit.autopsy.ingest; +import java.util.HashMap; + /** * An adapter that provides a default implementation of the IngestModule * interface. */ public abstract class IngestModuleAdapter implements IngestModule { + // Maps a JobId to the count of instances + static HashMap moduleRefCount = new HashMap<>(); + + public static synchronized void moduleRefCountIncrement(long jobID) { + long count = moduleRefCount.containsKey(jobID) ? moduleRefCount.get(jobID) : 0; + moduleRefCount.put(jobID, count + 1); + } + + public static synchronized long moduleRefCountDecrementAndGet(long jobID) { + if (moduleRefCount.containsKey(jobID)) { + long count = moduleRefCount.get(jobID); + moduleRefCount.put(jobID, --count); + return count; + } else { + return 0; + } + } @Override public void startUp(IngestJobContext context) throws IngestModuleException { diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index f8f2a00cc1..98aca11b72 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.hashdatabase; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; @@ -57,27 +56,10 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges private List knownBadHashSets = new ArrayList<>(); private List knownHashSets = new ArrayList<>(); private long jobID; - // Maps a JobId to the count of instances - static HashMap moduleRefCount = new HashMap<>(); static AtomicLong totalKnownBadCount = new AtomicLong(0); static AtomicLong totalCalctime = new AtomicLong(0); static AtomicLong totalLookuptime = new AtomicLong(0); - - private static synchronized void moduleRefCountIncrement(long jobID) { - long count = moduleRefCount.containsKey(jobID) ? moduleRefCount.get(jobID) : 0; - moduleRefCount.put(jobID, count + 1); - } - - private static synchronized long moduleRefCountDecrementAndGet(long jobID) { - if (moduleRefCount.containsKey(jobID)) { - long count = moduleRefCount.get(jobID); - moduleRefCount.put(jobID, --count); - return count; - } else { - return 0; - } - } - + HashDbIngestModule(HashLookupModuleSettings settings) { this.settings = settings; } @@ -85,7 +67,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges @Override public void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context) throws IngestModuleException { jobID = context.getJobId(); - moduleRefCountIncrement(jobID); + IngestModuleAdapter.moduleRefCountIncrement(jobID); getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); if (knownBadHashSets.isEmpty()) { services.postMessage(IngestMessage.createWarningMessage( @@ -317,7 +299,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges @Override public void shutDown(boolean ingestJobCancelled) { - if (moduleRefCountDecrementAndGet(jobID) == 0) { + if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobID) == 0) { if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) { StringBuilder detailsSb = new StringBuilder(); //details From e76573ee8a74347889f3b7ad0841c7c07755f481 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 8 Apr 2014 16:42:01 -0400 Subject: [PATCH 02/17] Adding the missing bundle properties to this branch since my pull request to develop is still waiting. --- .../org/sleuthkit/autopsy/keywordsearch/Bundle.properties | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 040e9e0c03..a9a00a94b5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -263,3 +263,10 @@ KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1 minute (faster KeywordSearchGlobalSearchSettingsPanel.chunksLabel.text=Chunks in keyword index: KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5 minutes (overall ingest time will be longer) KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5 minutes (default) +KeywordSearchIngestModule.regExpHitLbl=Reg Ex hit: +KeywordSearchIngestModule.kwHitLbl=Keyword hit: +KeywordSearchIngestModule.kwHitThLbl=Keyword +KeywordSearchIngestModule.previewThLbl=Preview +KeywordSearchIngestModule.fileThLbl=File +KeywordSearchIngestModule.listThLbl=List +KeywordSearchIngestModule.regExThLbl=Reg Ex \ No newline at end of file From ada3dd732bdaac82caeac19136cee2a1a7d3e21f Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 8 Apr 2014 16:49:51 -0400 Subject: [PATCH 03/17] The renamed prop. --- .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index a8d124934c..2641f8b322 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -916,7 +916,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme detailsSb.append(""); //hit detailsSb.append(""); - detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLThLbl")); + detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitThLbl")); detailsSb.append(""); detailsSb.append(""); From 42e83d3b23a3f08bec8a680b39d3d57b321fcdb2 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 8 Apr 2014 17:42:30 -0400 Subject: [PATCH 04/17] ingestStatus is now static and shared between all threads --- .../KeywordSearchIngestModule.java | 89 ++++++++++--------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index f6897510b4..ba30cc097f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -91,7 +91,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private long dataSourceId; private static AtomicInteger instanceCount = new AtomicInteger(0); //just used for logging private int instanceNum = 0; - + private enum IngestStatus { TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested @@ -101,8 +101,14 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme SKIPPED_ERROR_TEXTEXTRACT, ///< File was skipped because of text extraction issues SKIPPED_ERROR_IO ///< File was skipped because of IO issues reading it }; - private Map ingestStatus; + private static final Map ingestStatus = new HashMap<>(); //guarded by itself + static void putIngestStatus(long id, IngestStatus status) { + synchronized(ingestStatus) { + ingestStatus.put(id, status); + } + } + KeywordSearchIngestModule(KeywordSearchJobSettings settings) { this.settings = settings; instanceNum = instanceCount.getAndIncrement(); @@ -167,8 +173,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme textExtractors.add(new AbstractFileHtmlExtract(this)); textExtractors.add(new AbstractFileTikaTextExtract(this)); - ingestStatus = new HashMap<>(); - List keywordLists = KeywordSearchListsXML.getCurrent().getListsL(); boolean hasKeywordsForSearch = false; for (KeywordList keywordList : keywordLists) { @@ -191,7 +195,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme if (initialized == false) //error initializing indexing/Solr { logger.log(Level.WARNING, "Skipping processing, module not initialized, file: {0}", abstractFile.getName()); - ingestStatus.put(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); return ProcessResult.OK; } try { @@ -276,7 +280,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme * Common cleanup code when module stops or final searcher completes */ private void cleanup() { - ingestStatus.clear(); + synchronized(ingestStatus) { + ingestStatus.clear(); + } textExtractors.clear(); textExtractors = null; @@ -297,31 +303,34 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme int error_text = 0; int error_index = 0; int error_io = 0; - for (IngestStatus s : ingestStatus.values()) { - switch (s) { - case TEXT_INGESTED: - ++text_ingested; - break; - case METADATA_INGESTED: - ++metadata_ingested; - break; - case STRINGS_INGESTED: - ++strings_ingested; - break; - case SKIPPED_ERROR_TEXTEXTRACT: - error_text++; - break; - case SKIPPED_ERROR_INDEXING: - error_index++; - break; - case SKIPPED_ERROR_IO: - error_io++; - break; - default: - ; + + synchronized(ingestStatus) { + for (IngestStatus s : ingestStatus.values()) { + switch (s) { + case TEXT_INGESTED: + ++text_ingested; + break; + case METADATA_INGESTED: + ++metadata_ingested; + break; + case STRINGS_INGESTED: + ++strings_ingested; + break; + case SKIPPED_ERROR_TEXTEXTRACT: + error_text++; + break; + case SKIPPED_ERROR_INDEXING: + error_index++; + break; + case SKIPPED_ERROR_IO: + error_io++; + break; + default: + ; + } } } - + StringBuilder msg = new StringBuilder(); msg.append("
").append(EscapeUtil.escapeHtml(attr.getValueString())).append("
"); msg.append(""); @@ -393,16 +402,16 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private boolean extractStringsAndIndex(AbstractFile aFile) { try { if (stringExtractor.index(aFile)) { - ingestStatus.put(aFile.getId(), IngestStatus.STRINGS_INGESTED); + putIngestStatus(aFile.getId(), IngestStatus.STRINGS_INGESTED); return true; } else { logger.log(Level.WARNING, "Failed to extract strings and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()}); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); return false; } } catch (IngesterException ex) { logger.log(Level.WARNING, "Failed to extract strings and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").", ex); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); return false; } } @@ -448,9 +457,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme if ((indexContent == false || aFile.isDir() || size == 0)) { try { ingester.ingest(aFile, false); //meta-data only - ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED); + putIngestStatus(aFile.getId(), IngestStatus.METADATA_INGESTED); } catch (IngesterException ex) { - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex); } return; @@ -484,9 +493,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme if (AbstractFileExtract.ARCHIVE_MIME_TYPES.contains(detectedFormat)) { try { ingester.ingest(aFile, false); //meta-data only - ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED); + putIngestStatus(aFile.getId(), IngestStatus.METADATA_INGESTED); } catch (IngesterException ex) { - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex); } return; @@ -499,20 +508,20 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme //logger.log(Level.INFO, "indexing: " + aFile.getName()); if (!extractTextAndIndex(aFile, detectedFormat)) { logger.log(Level.WARNING, "Failed to extract text and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()}); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); } else { - ingestStatus.put(aFile.getId(), IngestStatus.TEXT_INGESTED); + putIngestStatus(aFile.getId(), IngestStatus.TEXT_INGESTED); wasTextAdded = true; } } catch (IngesterException e) { logger.log(Level.INFO, "Could not extract text with Tika, " + aFile.getId() + ", " + aFile.getName(), e); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); } catch (Exception e) { logger.log(Level.WARNING, "Error extracting text with Tika, " + aFile.getId() + ", " + aFile.getName(), e); - ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); + putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); } } From 000a70a7d9a8be978ac8f7ed0425858902735501 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 8 Apr 2014 17:48:21 -0400 Subject: [PATCH 05/17] Just do one summary postMessage for KeywordSearch module. --- .../autopsy/keywordsearch/KeywordSearchIngestModule.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index ba30cc097f..d19e0ea328 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -125,6 +125,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme initialized = false; jobId = context.getJobId(); + IngestModuleAdapter.moduleRefCountIncrement(jobId); caseHandle = Case.getCurrentCase().getSleuthkitCase(); tikaFormatDetector = new Tika(); ingester = Server.getIngester(); @@ -251,7 +252,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme // Remove from the search list and trigger final commit and final search SearchRunner.getInstance().endJob(jobId); - postIndexSummary(); + if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { + postIndexSummary(); + } //log number of files / chunks in index //signal a potential change in number of text_ingested files From 4be1592f955fca3d964a7bfb24cdda034ed87505 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Tue, 8 Apr 2014 17:48:35 -0400 Subject: [PATCH 06/17] renaming --- .../autopsy/ingest/IngestModuleAdapter.java | 14 +++++++------- .../autopsy/hashdatabase/HashDbIngestModule.java | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java index 1c9d1f0aa8..c2c4baa1ee 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java @@ -28,15 +28,15 @@ public abstract class IngestModuleAdapter implements IngestModule { // Maps a JobId to the count of instances static HashMap moduleRefCount = new HashMap<>(); - public static synchronized void moduleRefCountIncrement(long jobID) { - long count = moduleRefCount.containsKey(jobID) ? moduleRefCount.get(jobID) : 0; - moduleRefCount.put(jobID, count + 1); + public static synchronized void moduleRefCountIncrement(long jobId) { + long count = moduleRefCount.containsKey(jobId) ? moduleRefCount.get(jobId) : 0; + moduleRefCount.put(jobId, count + 1); } - public static synchronized long moduleRefCountDecrementAndGet(long jobID) { - if (moduleRefCount.containsKey(jobID)) { - long count = moduleRefCount.get(jobID); - moduleRefCount.put(jobID, --count); + public static synchronized long moduleRefCountDecrementAndGet(long jobId) { + if (moduleRefCount.containsKey(jobId)) { + long count = moduleRefCount.get(jobId); + moduleRefCount.put(jobId, --count); return count; } else { return 0; diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index 98aca11b72..473884a3a8 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -55,7 +55,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges private final HashLookupModuleSettings settings; private List knownBadHashSets = new ArrayList<>(); private List knownHashSets = new ArrayList<>(); - private long jobID; + private long jobId; static AtomicLong totalKnownBadCount = new AtomicLong(0); static AtomicLong totalCalctime = new AtomicLong(0); static AtomicLong totalLookuptime = new AtomicLong(0); @@ -66,8 +66,8 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges @Override public void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context) throws IngestModuleException { - jobID = context.getJobId(); - IngestModuleAdapter.moduleRefCountIncrement(jobID); + jobId = context.getJobId(); + IngestModuleAdapter.moduleRefCountIncrement(jobId); getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); if (knownBadHashSets.isEmpty()) { services.postMessage(IngestMessage.createWarningMessage( @@ -299,7 +299,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges @Override public void shutDown(boolean ingestJobCancelled) { - if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobID) == 0) { + if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) { StringBuilder detailsSb = new StringBuilder(); //details From e740ade703153f0b3e29191786bc3c7b0b338943 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 10:49:21 -0400 Subject: [PATCH 07/17] HashDB module: only post error messages once per job --- .../autopsy/ingest/IngestModuleAdapter.java | 6 ++- .../hashdatabase/HashDbIngestModule.java | 40 ++++++++++--------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java index c2c4baa1ee..101ea80dae 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java @@ -28,9 +28,11 @@ public abstract class IngestModuleAdapter implements IngestModule { // Maps a JobId to the count of instances static HashMap moduleRefCount = new HashMap<>(); - public static synchronized void moduleRefCountIncrement(long jobId) { + public static synchronized long moduleRefCountIncrement(long jobId) { long count = moduleRefCount.containsKey(jobId) ? moduleRefCount.get(jobId) : 0; - moduleRefCount.put(jobId, count + 1); + long nextCount = count + 1; + moduleRefCount.put(jobId, nextCount); + return nextCount; } public static synchronized long moduleRefCountDecrementAndGet(long jobId) { diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index 473884a3a8..97cf92b32f 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -66,26 +66,30 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges @Override public void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context) throws IngestModuleException { - jobId = context.getJobId(); - IngestModuleAdapter.moduleRefCountIncrement(jobId); + jobId = context.getJobId(); getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); - if (knownBadHashSets.isEmpty()) { - services.postMessage(IngestMessage.createWarningMessage( - HashLookupModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.noKnownBadHashDbSetMsg"), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn"))); - } - getEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets); - if (knownHashSets.isEmpty()) { - services.postMessage(IngestMessage.createWarningMessage( - HashLookupModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.noKnownHashDbSetMsg"), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.knownFileSearchWillNotExecuteWarn"))); + + if (IngestModuleAdapter.moduleRefCountIncrement(jobId) == 1) { + // if first module for this job then post error msgs if needed + + if (knownBadHashSets.isEmpty()) { + services.postMessage(IngestMessage.createWarningMessage( + HashLookupModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.noKnownBadHashDbSetMsg"), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn"))); + } + + if (knownHashSets.isEmpty()) { + services.postMessage(IngestMessage.createWarningMessage( + HashLookupModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.noKnownHashDbSetMsg"), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.knownFileSearchWillNotExecuteWarn"))); + } } } From d8d03aeae1d099d1dea82e3177c108e9c09cef54 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 11:00:36 -0400 Subject: [PATCH 08/17] KeywordSearch: check the server and existence of keywords only once per job --- .../KeywordSearchIngestModule.java | 79 ++++++++++--------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index d19e0ea328..06bdec668b 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -122,41 +122,57 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme @Override public void startUp(IngestJobContext context) throws IngestModuleException { logger.log(Level.INFO, "Initializing instance {0}", instanceNum); - initialized = false; - + initialized = false; jobId = context.getJobId(); - IngestModuleAdapter.moduleRefCountIncrement(jobId); caseHandle = Case.getCurrentCase().getSleuthkitCase(); tikaFormatDetector = new Tika(); ingester = Server.getIngester(); - final Server server = KeywordSearch.getServer(); - try { - if (!server.isRunning()) { + if (IngestModuleAdapter.moduleRefCountIncrement(jobId) == 1) { + // if first module for this job then check the server and existence of keywords + + final Server server = KeywordSearch.getServer(); + try { + if (!server.isRunning()) { + String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); + logger.log(Level.SEVERE, msg); + String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); + services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details)); + throw new IngestModuleException(msg); + } + } catch (KeywordSearchModuleException ex) { + logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex); + //this means Solr is not properly initialized String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); - logger.log(Level.SEVERE, msg); String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details)); throw new IngestModuleException(msg); } - } catch (KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex); - //this means Solr is not properly initialized - String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); - String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); - services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details)); - throw new IngestModuleException(msg); - } - try { - // make an actual query to verify that server is responding - // we had cases where getStatus was OK, but the connection resulted in a 404 - server.queryNumIndexedDocuments(); - } catch (KeywordSearchModuleException | NoOpenCoreException ex) { - throw new IngestModuleException( - NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg", - ex.getMessage())); - } + try { + // make an actual query to verify that server is responding + // we had cases where getStatus was OK, but the connection resulted in a 404 + server.queryNumIndexedDocuments(); + } catch (KeywordSearchModuleException | NoOpenCoreException ex) { + throw new IngestModuleException( + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg", + ex.getMessage())); + } + // check if this job has any searchable keywords + List keywordLists = KeywordSearchListsXML.getCurrent().getListsL(); + boolean hasKeywordsForSearch = false; + for (KeywordList keywordList : keywordLists) { + if (settings.isKeywordListEnabled(keywordList.getName()) && !keywordList.getKeywords().isEmpty()) { + hasKeywordsForSearch = true; + break; + } + } + if (!hasKeywordsForSearch) { + services.postMessage(IngestMessage.createWarningMessage(KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"), + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg"))); + } + } + //initialize extractors stringExtractor = new AbstractFileStringExtract(this); stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); @@ -173,20 +189,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme //order matters, more specific extractors first textExtractors.add(new AbstractFileHtmlExtract(this)); textExtractors.add(new AbstractFileTikaTextExtract(this)); - - List keywordLists = KeywordSearchListsXML.getCurrent().getListsL(); - boolean hasKeywordsForSearch = false; - for (KeywordList keywordList : keywordLists) { - if (settings.isKeywordListEnabled(keywordList.getName()) && !keywordList.getKeywords().isEmpty()) { - hasKeywordsForSearch = true; - break; - } - } - if (!hasKeywordsForSearch) { - services.postMessage(IngestMessage.createWarningMessage(KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"), - NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg"))); - } - + indexer = new Indexer(); initialized = true; } From 85451734a7abf0cc3bd03d3b6341f6d3bbeb0d5f Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 11:08:23 -0400 Subject: [PATCH 09/17] remove unused messageId var --- .../thunderbirdparser/ThunderbirdMboxFileIngestModule.java | 1 - 1 file changed, 1 deletion(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 2389b2e0f4..289f1fa5a0 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -54,7 +54,6 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); private IngestServices services = IngestServices.getInstance(); - private int messageId = 0; // RJCTODO: Not thread safe private FileManager fileManager; private IngestJobContext context; From 09376f73d4ce225eae446f0b3556e1ce953f9143 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 12:25:45 -0400 Subject: [PATCH 10/17] multithreading updates for ExifParserFileIngestModule --- .../exif/ExifParserFileIngestModule.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index 3df8d1686e..d12c63cff4 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -32,11 +32,13 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.Date; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; @@ -57,12 +59,20 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem private static final Logger logger = Logger.getLogger(ExifParserFileIngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); - private int filesProcessed = 0; - private boolean filesToFire = false; + private AtomicInteger filesProcessed = new AtomicInteger(0); + private volatile boolean filesToFire = false; + private long jobId; ExifParserFileIngestModule() { } + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + jobId = context.getJobId(); + IngestModuleAdapter.moduleRefCountIncrement(jobId); + } + + @Override public ProcessResult process(AbstractFile content) { //skip unalloc @@ -76,8 +86,8 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem } // update the tree every 1000 files if we have EXIF data that is not being being displayed - filesProcessed++; - if ((filesToFire) && (filesProcessed % 1000 == 0)) { + final int filesProcessedValue = filesProcessed.incrementAndGet(); + if ((filesToFire) && (filesProcessedValue % 1000 == 0)) { services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); filesToFire = false; } @@ -187,9 +197,12 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem @Override public void shutDown(boolean ingestJobCancelled) { - if (filesToFire) { - //send the final new data event - services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); + // We only need to check for this final event on the last thread per job + if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { + if (filesToFire) { + //send the final new data event + services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); + } } } } \ No newline at end of file From b0b9e8201ccbee6463ef0f3129917f47dc0c580e Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 13:32:58 -0400 Subject: [PATCH 11/17] FileExtMismatchIngestModule: only send one final message --- .../exif/ExifParserFileIngestModule.java | 2 +- .../FileExtMismatchIngestModule.java | 44 +++++++++++-------- .../KeywordSearchIngestModule.java | 1 + 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index d12c63cff4..1fa1250960 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -197,7 +197,7 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem @Override public void shutDown(boolean ingestJobCancelled) { - // We only need to check for this final event on the last thread per job + // We only need to check for this final event on the last module per job if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { if (filesToFire) { //send the final new data event diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java index a04686bded..f7bb46f95f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; @@ -38,7 +39,6 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.datamodel.TskException; /** @@ -50,8 +50,9 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements private final IngestServices services = IngestServices.getInstance(); private final FileExtMismatchDetectorModuleSettings settings; private HashMap SigTypeToExtMap = new HashMap<>(); - private long processTime = 0; - private long numFiles = 0; + private long jobId; + private static AtomicLong processTime = new AtomicLong(0); + private static AtomicLong numFiles = new AtomicLong(0); FileExtMismatchIngestModule(FileExtMismatchDetectorModuleSettings settings) { this.settings = settings; @@ -59,6 +60,8 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements @Override public void startUp(IngestJobContext context) throws IngestModuleException { + jobId = context.getJobId(); + IngestModuleAdapter.moduleRefCountIncrement(jobId); FileExtMismatchXML xmlLoader = FileExtMismatchXML.getDefault(); SigTypeToExtMap = xmlLoader.load(); } @@ -82,8 +85,8 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements boolean mismatchDetected = compareSigTypeToExt(abstractFile); - processTime += (System.currentTimeMillis() - startTime); - numFiles++; + processTime.getAndAdd(System.currentTimeMillis() - startTime); + numFiles.getAndIncrement(); if (mismatchDetected) { // add artifact @@ -149,19 +152,22 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements @Override public void shutDown(boolean ingestJobCancelled) { - StringBuilder detailsSb = new StringBuilder(); - detailsSb.append("
").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.knowFileHeaderLbl")).append("").append(text_ingested).append("
").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.fileGenStringsHead")).append("").append(strings_ingested).append("
"); - detailsSb.append(""); - detailsSb.append("\n"); - detailsSb.append("\n"); - detailsSb.append("
").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("
").append( - NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalProcTime")) - .append("").append(processTime).append("
").append( - NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalFiles")) - .append("").append(numFiles).append("
"); - services.postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileExtMismatchDetectorModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "FileExtMismatchIngestModule.complete.svcMsg.text"), - detailsSb.toString())); + // We only need to post the summary msg from the last module per job + if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { + StringBuilder detailsSb = new StringBuilder(); + detailsSb.append(""); + detailsSb.append(""); + detailsSb.append("\n"); + detailsSb.append("\n"); + detailsSb.append("
").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("
").append( + NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalProcTime")) + .append("").append(processTime.get()).append("
").append( + NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalFiles")) + .append("").append(numFiles.get()).append("
"); + services.postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileExtMismatchDetectorModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "FileExtMismatchIngestModule.complete.svcMsg.text"), + detailsSb.toString())); + } } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 06bdec668b..ad903141ed 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -255,6 +255,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme // Remove from the search list and trigger final commit and final search SearchRunner.getInstance().endJob(jobId); + // We only need to post the summary msg from the last module per job if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { postIndexSummary(); } From be4f3194fc7970c9d4fcb4787212d59341e57f8b Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 13:57:48 -0400 Subject: [PATCH 12/17] FileTypeIdIngestModule: only send one final message --- .../filetypeid/FileTypeIdIngestModule.java | 52 ++++++++++++------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 8e0c790656..48e1a997f4 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -18,11 +18,12 @@ */ package org.sleuthkit.autopsy.modules.filetypeid; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; @@ -44,8 +45,10 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName()); private static final long MIN_FILE_SIZE = 512; private final FileTypeIdModuleSettings settings; - private long matchTime = 0; - private long numFiles = 0; + private long jobId; + private static AtomicLong matchTime = new AtomicLong(0); + private static AtomicLong numFiles = new AtomicLong(0); + // The detector. Swap out with a different implementation of FileTypeDetectionInterface as needed. // If desired in the future to be more knowledgable about weird files or rare formats, we could // actually have a list of detectors which are called in order until a match is found. @@ -55,6 +58,12 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI this.settings = settings; } + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + jobId = context.getJobId(); + IngestModuleAdapter.moduleRefCountIncrement(jobId); + } + @Override public ProcessResult process(AbstractFile abstractFile) { // skip non-files @@ -75,8 +84,8 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI try { long startTime = System.currentTimeMillis(); FileTypeDetectionInterface.FileIdInfo fileId = detector.attemptMatch(abstractFile); - matchTime += (System.currentTimeMillis() - startTime); - numFiles++; + matchTime.getAndAdd(System.currentTimeMillis() - startTime); + numFiles.getAndIncrement(); if (!fileId.type.isEmpty()) { // add artifact @@ -98,22 +107,25 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI @Override public void shutDown(boolean ingestJobCancelled) { - StringBuilder detailsSb = new StringBuilder(); - detailsSb.append(""); - detailsSb.append(""); - detailsSb.append("\n"); - detailsSb.append("\n"); - detailsSb.append("
").append(FileTypeIdModuleFactory.getModuleName()).append("
") - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) - .append("").append(matchTime).append("
") - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) - .append("").append(numFiles).append("
"); - IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), - "FileTypeIdIngestModule.complete.srvMsg.text"), - detailsSb.toString())); + // We only need to post the summary msg from the last module per job + if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { + StringBuilder detailsSb = new StringBuilder(); + detailsSb.append(""); + detailsSb.append(""); + detailsSb.append("\n"); + detailsSb.append("\n"); + detailsSb.append("
").append(FileTypeIdModuleFactory.getModuleName()).append("
") + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) + .append("").append(matchTime.get()).append("
") + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) + .append("").append(numFiles.get()).append("
"); + IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "FileTypeIdIngestModule.complete.srvMsg.text"), + detailsSb.toString())); + } } - + /** * Validate if a given mime type is in the detector's registry. * From 1e255f09794ebc0402f73d07b012d95dc6cbaff2 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 14:07:05 -0400 Subject: [PATCH 13/17] SevenZipIngestModule: only check 7zip init once --- .../sevenzip/SevenZipIngestModule.java | 31 +++++++++++-------- .../KeywordSearchIngestModule.java | 4 +-- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index 8bfc126cef..76ffa70926 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -90,6 +90,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F private final byte[] fileHeaderBuffer = new byte[readHeaderSize]; private static final int ZIP_SIGNATURE_BE = 0x504B0304; private IngestJobContext context; + private long jobId; SevenZipIngestModule() { } @@ -97,6 +98,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F @Override public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; + jobId = context.getJobId(); final Case currentCase = Case.getCurrentCase(); @@ -121,20 +123,23 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F } } - try { - SevenZip.initSevenZipFromPlatformJAR(); - String platform = SevenZip.getUsedPlatform(); - logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform); - } catch (SevenZipNativeInitializationException e) { - logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e); - String msg = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errInitModule.msg", - ArchiveFileExtractorModuleFactory.getModuleName()); - String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errCantInitLib", - e.getMessage()); - services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); - throw new RuntimeException(e); + // if first instance of this module for this job then check 7zip init + if (IngestModuleAdapter.moduleRefCountIncrement(jobId) == 1) { + try { + SevenZip.initSevenZipFromPlatformJAR(); + String platform = SevenZip.getUsedPlatform(); + logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform); + } catch (SevenZipNativeInitializationException e) { + logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e); + String msg = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errInitModule.msg", + ArchiveFileExtractorModuleFactory.getModuleName()); + String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errCantInitLib", + e.getMessage()); + services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); + throw new RuntimeException(e); + } } - + archiveDepthCountTree = new ArchiveDepthCountTree(); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index ad903141ed..fb7b5cc3d5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -128,9 +128,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme tikaFormatDetector = new Tika(); ingester = Server.getIngester(); + // increment the module reference count + // if first instance of this module for this job then check the server and existence of keywords if (IngestModuleAdapter.moduleRefCountIncrement(jobId) == 1) { - // if first module for this job then check the server and existence of keywords - final Server server = KeywordSearch.getServer(); try { if (!server.isRunning()) { From 2f098d2202e6c2e6a70abfaa67897e9db10d6eef Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 14:11:35 -0400 Subject: [PATCH 14/17] made a string final --- .../sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java b/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java index 5a0bc208a6..875dcaa772 100644 --- a/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java +++ b/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java @@ -53,7 +53,7 @@ class ScalpelCarverIngestModule extends IngestModuleAdapter implements FileInges private static final Logger logger = Logger.getLogger(ScalpelCarverIngestModule.class.getName()); private final String MODULE_OUTPUT_DIR_NAME = "ScalpelCarver"; private String moduleOutputDirPath; - private String configFileName = "scalpel.conf"; + private final String configFileName = "scalpel.conf"; private String configFilePath; private boolean initialized = false; private ScalpelCarver carver; From e2d66535abcc3515a2fef6b95eed7bc4520c0277 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 15:14:48 -0400 Subject: [PATCH 15/17] fix qualifier consistency issues with the table model --- .../autopsy/ingest/IngestMessagePanel.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java index 7596e1f4cd..dd8314f41e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java @@ -395,18 +395,18 @@ class IngestMessagePanel extends JPanel implements TableModelListener { } @Override - synchronized public int getRowCount() { + public synchronized int getRowCount() { return getNumberGroups(); } - public void markAllSeen() { + public synchronized void markAllSeen() { for (TableEntry entry : messageData) { entry.hasBeenSeen(true); } fireTableChanged(new TableModelEvent(this)); } - public int getNumberNewMessages() { + public synchronized int getNumberNewMessages() { int newMessages = 0; for (TableEntry entry : messageData) { if (!entry.hasBeenSeen()) { @@ -416,11 +416,11 @@ class IngestMessagePanel extends JPanel implements TableModelListener { return newMessages; } - synchronized int getNumberGroups() { + public synchronized int getNumberGroups() { return messageData.size(); } - synchronized int getNumberMessages() { + public synchronized int getNumberMessages() { int total = 0; for (TableEntry e : messageData) { total += e.messageGroup.getCount(); @@ -428,7 +428,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { return total; } - synchronized int getNumberUnreadMessages() { + public synchronized int getNumberUnreadMessages() { int total = 0; for (TableEntry e : messageData) { if (!e.hasBeenVisited) { @@ -438,7 +438,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { return total; } - synchronized int getNumberUnreadGroups() { + public synchronized int getNumberUnreadGroups() { int total = 0; for (TableEntry e : messageData) { if (!e.hasBeenVisited) { @@ -513,7 +513,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { return ret; } - synchronized public void addMessage(IngestMessage m) { + public synchronized void addMessage(IngestMessage m) { //check how many messages per module with the same uniqness //and add to existing group or create a new group String moduleName = m.getSource(); @@ -628,15 +628,27 @@ class IngestMessagePanel extends JPanel implements TableModelListener { } public synchronized boolean isVisited(int rowNumber) { - return messageData.get(rowNumber).hasBeenVisited(); + if (rowNumber < messageData.size()) { + return messageData.get(rowNumber).hasBeenVisited(); + } else { + return false; + } } public synchronized MessageType getMessageType(int rowNumber) { - return messageData.get(rowNumber).messageGroup.getMessageType(); + if (rowNumber < messageData.size()) { + return messageData.get(rowNumber).messageGroup.getMessageType(); + } else { + return null; + } } public synchronized IngestMessageGroup getMessageGroup(int rowNumber) { - return messageData.get(rowNumber).messageGroup; + if (rowNumber < messageData.size()) { + return messageData.get(rowNumber).messageGroup; + } else { + return null; + } } public synchronized void reSort(boolean chronoLogical) { From 7e7f7b43bb453624e5b341945598a928e79df09f Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 17:35:18 -0400 Subject: [PATCH 16/17] updates to make IngestMessagePanel table model thread-safe --- .../autopsy/ingest/IngestMessagePanel.java | 88 ++++++++++--------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java index dd8314f41e..74a0901004 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java @@ -713,26 +713,26 @@ class IngestMessagePanel extends JPanel implements TableModelListener { messages.add(message); } - List getMessages() { + synchronized List getMessages() { return messages; } - void add(IngestMessage message) { + synchronized void add(IngestMessage message) { messages.add(message); } //add all messages from another group - void addAll(IngestMessageGroup group) { + synchronized void addAll(IngestMessageGroup group) { for (IngestMessage m : group.getMessages()) { messages.add(m); } } - int getCount() { + synchronized int getCount() { return messages.size(); } - String getDetails() { + synchronized String getDetails() { StringBuilder b = new StringBuilder(""); for (IngestMessage m : messages) { String details = m.getDetails(); @@ -751,7 +751,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { * return color corresp to priority * @return */ - Color getColor() { + synchronized Color getColor() { int count = messages.size(); if (count == 1) { return VERY_HIGH_PRI_COLOR; @@ -769,7 +769,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { * used for chrono sort * @return */ - Date getDatePosted() { + synchronized Date getDatePosted() { return messages.get(messages.size() - 1).getDatePosted(); } @@ -777,35 +777,35 @@ class IngestMessagePanel extends JPanel implements TableModelListener { * get subject of the first message * @return */ - String getSubject() { + synchronized String getSubject() { return messages.get(0).getSubject(); } /* * return unique key, should be the same for all msgs */ - String getUniqueKey() { + synchronized String getUniqueKey() { return messages.get(0).getUniqueKey(); } /* * return source module, should be the same for all msgs */ - String getSource() { + synchronized String getSource() { return messages.get(0).getSource(); } /* * return data of the first message */ - BlackboardArtifact getData() { + synchronized BlackboardArtifact getData() { return messages.get(0).getData(); } /* * return message type, should be the same for all msgs */ - IngestMessage.MessageType getMessageType() { + synchronized IngestMessage.MessageType getMessageType() { return messages.get(0).getMessageType(); } } @@ -870,16 +870,17 @@ class IngestMessagePanel extends JPanel implements TableModelListener { cell.setFont(new Font("", Font.PLAIN, 16)); final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row); - MessageType mt = messageGroup.getMessageType(); - if (mt == MessageType.ERROR) { - cell.setBackground(ERROR_COLOR); - } else if (mt == MessageType.WARNING) { - cell.setBackground(Color.orange); - } else { - //cell.setBackground(table.getBackground()); - cell.setBackground(messageGroup.getColor()); + if (messageGroup != null) { + MessageType mt = messageGroup.getMessageType(); + if (mt == MessageType.ERROR) { + cell.setBackground(ERROR_COLOR); + } else if (mt == MessageType.WARNING) { + cell.setBackground(Color.orange); + } else { + //cell.setBackground(table.getBackground()); + cell.setBackground(messageGroup.getColor()); + } } - return cell; } } @@ -910,16 +911,17 @@ class IngestMessagePanel extends JPanel implements TableModelListener { } final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row); - MessageType mt = messageGroup.getMessageType(); - if (mt == MessageType.ERROR) { - cell.setBackground(ERROR_COLOR); - } else if (mt == MessageType.WARNING) { - cell.setBackground(Color.orange); - } else { - //cell.setBackground(table.getBackground()); - cell.setBackground(messageGroup.getColor()); + if (messageGroup != null) { + MessageType mt = messageGroup.getMessageType(); + if (mt == MessageType.ERROR) { + cell.setBackground(ERROR_COLOR); + } else if (mt == MessageType.WARNING) { + cell.setBackground(Color.orange); + } else { + //cell.setBackground(table.getBackground()); + cell.setBackground(messageGroup.getColor()); + } } - return cell; } } @@ -945,14 +947,16 @@ class IngestMessagePanel extends JPanel implements TableModelListener { Component cell = super.getTableCellRendererComponent(table, aValue, isSelected, hasFocus, row, column); final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row); - MessageType mt = messageGroup.getMessageType(); - if (mt == MessageType.ERROR) { - cell.setBackground(ERROR_COLOR); - } else if (mt == MessageType.WARNING) { - cell.setBackground(Color.orange); - } else { - //cell.setBackground(table.getBackground()); - cell.setBackground(messageGroup.getColor()); + if (messageGroup != null) { + MessageType mt = messageGroup.getMessageType(); + if (mt == MessageType.ERROR) { + cell.setBackground(ERROR_COLOR); + } else if (mt == MessageType.WARNING) { + cell.setBackground(Color.orange); + } else { + //cell.setBackground(table.getBackground()); + cell.setBackground(messageGroup.getColor()); + } } return cell; @@ -986,9 +990,11 @@ class IngestMessagePanel extends JPanel implements TableModelListener { messageTable.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); //check if has details IngestMessageGroup m = getMessageGroup(selected); - String details = m.getDetails(); - if (details != null && !details.equals("")) { - mainPanel.showDetails(selected); + if (m != null) { + String details = m.getDetails(); + if (details != null && !details.equals("")) { + mainPanel.showDetails(selected); + } } messageTable.setCursor(null); } From dc5624916e1c71e82ca271c341cd2bd25b6c6ad2 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Wed, 9 Apr 2014 17:38:59 -0400 Subject: [PATCH 17/17] removed IngestMessageGroup.getMessages() --- .../org/sleuthkit/autopsy/ingest/IngestMessagePanel.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java index 74a0901004..36ffe035dd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java @@ -713,17 +713,13 @@ class IngestMessagePanel extends JPanel implements TableModelListener { messages.add(message); } - synchronized List getMessages() { - return messages; - } - synchronized void add(IngestMessage message) { messages.add(message); } //add all messages from another group synchronized void addAll(IngestMessageGroup group) { - for (IngestMessage m : group.getMessages()) { + for (IngestMessage m : messages) { messages.add(m); } }