From 7152dbc9c47fb06b81492e40ce6855699a0b1cb0 Mon Sep 17 00:00:00 2001 From: apriestman Date: Fri, 17 Dec 2021 16:55:22 -0500 Subject: [PATCH 1/2] Include ingest job in credit card account creation --- .../autopsy/ingest/IngestJobExecutor.java | 29 ++++++++++++++++--- .../keywordsearch/KeywordSearchQuery.java | 2 +- .../autopsy/keywordsearch/LuceneQuery.java | 8 ++--- .../autopsy/keywordsearch/QueryResults.java | 3 +- .../autopsy/keywordsearch/RegexQuery.java | 14 ++++----- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 77b3d026a4..84da5c53a8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -722,10 +722,31 @@ final class IngestJobExecutor { */ void addDataArtifacts(List artifacts) { if (!isCancelled() && ingestModuleTiers.get(moduleTierIndex).hasDataArtifactIngestModules()) { - if (jobState.equals(IngestJobState.ACCEPTING_STREAMED_CONTENT_AND_ANALYZING) || jobState.equals(IngestJobState.ANALYZING)) { - taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); - } else { - logErrorMessage(Level.SEVERE, "Attempt to add data artifacts to job during stage " + jobState.toString() + " not supported"); + switch (jobState) { + case ACCEPTING_STREAMED_CONTENT_AND_ANALYZING: + case ANALYZING: + taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); + break; + case PIPELINES_SHUTTING_DOWN: + /* + * Don't log an error if there is an attempt to add an + * data artifact ingest task in a pipeline shut down + * state. This is a work around for dealing with data + * artifacts generated by a final keyword search carried out + * during ingest module shut down by simply ignoring them. + * (Currently these are credit card accounts generated by + * keyword search). Other ideas were to add + * a startShutDown() phase to the ingest module + * life cycle (complicated), or to add a flag + * to keyword hit processing to suppress posting the keyword + * hit analysis results / data artifacts to the blackboard during a final + * search (API changes required to allow firing of the event + * to make any GUI refresh). + */ + break; + default: + logErrorMessage(Level.SEVERE, "Attempt to add data artifacts to job during stage " + jobState.toString() + " not supported"); + break; } } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java index ac005b8cc1..f1b8a25a41 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java @@ -116,5 +116,5 @@ interface KeywordSearchQuery { * @return The newly created artifact or null if there was a problem * creating it. */ - BlackboardArtifact createKeywordHitArtifact(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName); + BlackboardArtifact createKeywordHitArtifact(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName, Long ingestJobId); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index 43da17fa0b..3fcc77a315 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -36,12 +36,12 @@ import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskException; /** @@ -233,7 +233,7 @@ class LuceneQuery implements KeywordSearchQuery { * creating it. */ @Override - public BlackboardArtifact createKeywordHitArtifact(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { + public BlackboardArtifact createKeywordHitArtifact(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName, Long ingestJobId) { final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); Collection attributes = new ArrayList<>(); @@ -252,9 +252,9 @@ class LuceneQuery implements KeywordSearchQuery { } if (originalKeyword.searchTermIsWholeWord()) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.LITERAL.ordinal())); + attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.LITERAL.getType())); } else { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal())); + attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.SUBSTRING.getType())); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 5492f768d8..5ac61eb946 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -36,7 +36,6 @@ import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; -; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -194,7 +193,7 @@ class QueryResults { /* * Post an artifact for the hit to the blackboard. */ - BlackboardArtifact artifact = query.createKeywordHitArtifact(content, keyword, hit, snippet, query.getKeywordList().getName()); + BlackboardArtifact artifact = query.createKeywordHitArtifact(content, keyword, hit, snippet, query.getKeywordList().getName(), ingestJobId); /* * Send an ingest inbox message for the hit. diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java index 61ddc44921..b6c74b3da2 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java @@ -572,7 +572,7 @@ final class RegexQuery implements KeywordSearchQuery { * creating it. */ @Override - public BlackboardArtifact createKeywordHitArtifact(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { + public BlackboardArtifact createKeywordHitArtifact(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName, Long ingestJobId) { final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); if (content == null) { @@ -584,7 +584,7 @@ final class RegexQuery implements KeywordSearchQuery { * Credit Card number hits are handled differently */ if (originalKeyword.getArtifactAttributeType() == ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { - createCCNAccount(content, foundKeyword, hit, snippet, listName); + createCCNAccount(content, foundKeyword, hit, snippet, listName, ingestJobId); return null; } @@ -609,9 +609,9 @@ final class RegexQuery implements KeywordSearchQuery { ); if (originalKeyword.searchTermIsLiteral()) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.SUBSTRING.ordinal())); } else { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.REGEX.ordinal())); } try { @@ -625,7 +625,7 @@ final class RegexQuery implements KeywordSearchQuery { } } - private void createCCNAccount(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { + private void createCCNAccount(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName, Long ingestJobId) { final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); @@ -714,13 +714,13 @@ final class RegexQuery implements KeywordSearchQuery { -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) ); - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.REGEX.getType())); /* * Create an account instance. */ try { - AccountFileInstance ccAccountInstance = Case.getCurrentCaseThrows().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.CREDIT_CARD, ccnAttribute.getValueString(), MODULE_NAME, content); + AccountFileInstance ccAccountInstance = Case.getCurrentCaseThrows().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.CREDIT_CARD, ccnAttribute.getValueString(), MODULE_NAME, content, ingestJobId); ccAccountInstance.addAttributes(attributes); From 0cba3a6c2f630e640e0427b9f892d12e55e4768a Mon Sep 17 00:00:00 2001 From: apriestman Date: Fri, 17 Dec 2021 17:11:27 -0500 Subject: [PATCH 2/2] Revert enum changes --- .../org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java | 6 +++--- .../src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index 3fcc77a315..a07ddb66f9 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -36,12 +36,12 @@ import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskException; /** @@ -252,9 +252,9 @@ class LuceneQuery implements KeywordSearchQuery { } if (originalKeyword.searchTermIsWholeWord()) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.LITERAL.getType())); + attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.LITERAL.ordinal())); } else { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.SUBSTRING.getType())); + attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal())); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java index b6c74b3da2..c2b4189ae1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java @@ -609,9 +609,9 @@ final class RegexQuery implements KeywordSearchQuery { ); if (originalKeyword.searchTermIsLiteral()) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.SUBSTRING.ordinal())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal())); } else { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.REGEX.ordinal())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal())); } try { @@ -714,7 +714,7 @@ final class RegexQuery implements KeywordSearchQuery { -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) ); - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, TskData.KeywordSearchQueryType.REGEX.getType())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal())); /* * Create an account instance.