mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

7895 CR data artifact ingest module

Browse Source
This commit is contained in:
Richard Cordovano 2021-11-04 14:17:34 -04:00
parent 15702450e2
commit 826b772bb1
4 changed files with 143 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java
View File

@ -1,7 +1,7 @@
/* /*
* Central Repository * Central Repository
* *
* Copyright 2015-2019 Basis Technology Corp. * Copyright 2015-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org * Contact: carrier <at> sleuthkit <dot> org
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
@ -98,32 +98,33 @@ public class CorrelationDataSource implements Serializable {
} }
/** /**
* Create a CorrelationDataSource object from a TSK Content object. This * Creates a central repository data source object from a case database data
* will add it to the central repository. * source. If the data source is not already present in the central
* repository, it is added.
* *
* @param correlationCase the current CorrelationCase used for ensuring * @param correlationCase The central repository case associated with the
* uniqueness of DataSource * data aosurce.
* @param dataSource the sleuthkit datasource that is being added to * @param dataSource The case database data source.
* the central repository
* *
* @return * @return The cnetral repository data source.
* *
* @throws CentralRepoException * @throws CentralRepoException This exception is thrown if there is an
* error creating the central repository data
* source.
*/ */
public static CorrelationDataSource fromTSKDataSource(CorrelationCase correlationCase, Content dataSource) throws CentralRepoException { public static CorrelationDataSource fromTSKDataSource(CorrelationCase correlationCase, Content dataSource) throws CentralRepoException {
if (!CentralRepository.isEnabled()) {
throw new CentralRepoException("Central repository is not enabled, cannot create central repository data source, ");
}
Case curCase; Case curCase;
try { try {
curCase = Case.getCurrentCaseThrows(); curCase = Case.getCurrentCaseThrows();
} catch (NoCurrentCaseException ex) { } catch (NoCurrentCaseException ex) {
throw new CentralRepoException("Autopsy case is closed"); throw new CentralRepoException("Error getting current case", ex);
}
CorrelationDataSource correlationDataSource = null;
boolean useCR = CentralRepository.isEnabled();
if (useCR) {
correlationDataSource = CentralRepository.getInstance().getDataSource(correlationCase, dataSource.getId());
} }
CorrelationDataSource correlationDataSource = CentralRepository.getInstance().getDataSource(correlationCase, dataSource.getId());
if (correlationDataSource == null) { if (correlationDataSource == null) {
String deviceId; String deviceId;
String md5 = null; String md5 = null;
@ -131,7 +132,7 @@ public class CorrelationDataSource implements Serializable {
String sha256 = null; String sha256 = null;
try { try {
deviceId = curCase.getSleuthkitCase().getDataSource(dataSource.getId()).getDeviceId(); deviceId = curCase.getSleuthkitCase().getDataSource(dataSource.getId()).getDeviceId();
if (dataSource instanceof Image) { if (dataSource instanceof Image) {
Image image = (Image) dataSource; Image image = (Image) dataSource;
md5 = image.getMd5(); md5 = image.getMd5();
@ -139,15 +140,13 @@ public class CorrelationDataSource implements Serializable {
sha256 = image.getSha256(); sha256 = image.getSha256();
} }
} catch (TskDataException | TskCoreException ex) { } catch (TskDataException | TskCoreException ex) {
throw new CentralRepoException("Error getting data source info: " + ex.getMessage()); throw new CentralRepoException("Error getting data source info from case database", ex);
} }
correlationDataSource = new CorrelationDataSource(correlationCase, deviceId, dataSource.getName(), dataSource.getId(), md5, sha1, sha256); correlationDataSource = new CorrelationDataSource(correlationCase, deviceId, dataSource.getName(), dataSource.getId(), md5, sha1, sha256);
if (useCR) { correlationDataSource = CentralRepository.getInstance().newDataSource(correlationDataSource);
//add the correlation data source to the central repository and fill in the Central repository data source id in the object
correlationDataSource = CentralRepository.getInstance().newDataSource(correlationDataSource);
}
} }
return correlationDataSource; return correlationDataSource;
} }
@ -205,66 +204,68 @@ public class CorrelationDataSource implements Serializable {
public String getName() { public String getName() {
return name; return name;
} }
/** /**
* @return the MD5 hash value * @return the MD5 hash value
*/ */
public String getMd5() { public String getMd5() {
return (md5Hash == null ? "" : md5Hash); return (md5Hash == null ? "" : md5Hash);
} }
/** /**
* Set the MD5 hash value and persist to the Central Repository if available. * Set the MD5 hash value and persist to the Central Repository if
* * available.
*
* @param md5Hash The MD5 hash value. * @param md5Hash The MD5 hash value.
*
* @throws CentralRepoException If there's an issue updating the Central * @throws CentralRepoException If there's an issue updating the Central
Repository. * Repository.
*/ */
public void setMd5(String md5Hash) throws CentralRepoException { public void setMd5(String md5Hash) throws CentralRepoException {
this.md5Hash = md5Hash; this.md5Hash = md5Hash;
if (dataSourceObjectID != -1) { if (dataSourceObjectID != -1) {
CentralRepository.getInstance().updateDataSourceMd5Hash(this); CentralRepository.getInstance().updateDataSourceMd5Hash(this);
} }
} }
/** /**
* @return the SHA-1 hash value * @return the SHA-1 hash value
*/ */
public String getSha1() { public String getSha1() {
return (sha1Hash == null ? "" : sha1Hash); return (sha1Hash == null ? "" : sha1Hash);
} }
/** /**
* Set the SHA-1 hash value and persist to the Central Repository if * Set the SHA-1 hash value and persist to the Central Repository if
* available. * available.
* *
* @param sha1Hash The SHA-1 hash value. * @param sha1Hash The SHA-1 hash value.
*/ */
public void setSha1(String sha1Hash) throws CentralRepoException { public void setSha1(String sha1Hash) throws CentralRepoException {
this.sha1Hash = sha1Hash; this.sha1Hash = sha1Hash;
if (dataSourceObjectID != -1) { if (dataSourceObjectID != -1) {
CentralRepository.getInstance().updateDataSourceSha1Hash(this); CentralRepository.getInstance().updateDataSourceSha1Hash(this);
} }
} }
/** /**
* @return the SHA-256 hash value * @return the SHA-256 hash value
*/ */
public String getSha256() { public String getSha256() {
return (sha256Hash == null ? "" : sha256Hash); return (sha256Hash == null ? "" : sha256Hash);
} }
/** /**
* Set the SHA-256 hash value and persist to the Central Repository if * Set the SHA-256 hash value and persist to the Central Repository if
* available. * available.
* *
* @param sha256Hash The SHA-256 hash value. * @param sha256Hash The SHA-256 hash value.
*/ */
public void setSha256(String sha256Hash) throws CentralRepoException { public void setSha256(String sha256Hash) throws CentralRepoException {
this.sha256Hash = sha256Hash; this.sha256Hash = sha256Hash;
if (dataSourceObjectID != -1) { if (dataSourceObjectID != -1) {
CentralRepository.getInstance().updateDataSourceSha256Hash(this); CentralRepository.getInstance().updateDataSourceSha256Hash(this);
} }

32
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties-MERGED
View File

@ -1,27 +1,29 @@
CentralRepoIngestModel_name_header=Name:<br> CentralRepoIngestModel_name_header=Name:<br>
CentralRepoIngestModel_previous_case_header=<br>Previous Cases:<br> CentralRepoIngestModel_previous_case_header=<br>Previous Cases:<br>
CentralRepoIngestModule.errorMessage.isNotEnabled=Central repository settings are not initialized, cannot run Central Repository ingest module.
CentralRepoIngestModule.notfyBubble.title=Central Repository Not Initialized
CentralRepoIngestModule.prevCaseComment.text=Previous Case: CentralRepoIngestModule.prevCaseComment.text=Previous Case:
CentralRepoIngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository) CentralRepoIngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)
CentralRepoIngestModule_cannotGetCrCaseErrMsg=Case not present in the central repository
CentralRepoIngestModule_cannotGetCrDataSourceErrMsg=Data source not present in the central repository
CentralRepoIngestModule_crDatabaseTypeMismatch=Mulit-user cases require a PostgreSQL central repository
CentralRepoIngestModule_crInaccessibleErrMsg=Error accessing central repository
CentralRepoIngestModule_crNotEnabledErrMsg=Central repository required, but not enabled
CentralRepoIngestModule_missingFileCorrAttrTypeErrMsg=Correlation attribute type for files not found in the central repository
CentralRepoIngestModule_noCurrentCaseErrMsg=Error getting current case
CentralRepoIngestModule_notable_message_header=<html>A file in this data source was previously seen and tagged as Notable.<br> CentralRepoIngestModule_notable_message_header=<html>A file in this data source was previously seen and tagged as Notable.<br>
# {0} - list of cases
CentralRepoIngestModule_notableJustification=Previously marked as notable in cases {0}
CentralRepoIngestModule_notableSetName=Previously Tagged As Notable (Central Repository)
CentralRepoIngestModule_osAcctMgrInaccessibleErrMsg=Error getting OS accounts manager
# {0} - Name of file that is Notable # {0} - Name of file that is Notable
CentralRepoIngestModule_postToBB_knownBadMsg=Notable: {0} CentralRepoIngestModule_postToBB_knownBadMsg=Notable: {0}
# {0} - list of cases
CentralRepoIngestModule_prevSeenJustification=Previously seen in cases {0}
CentralRepoIngestModule_prevSeenOsAcctConfig=Previously Seen Users (Central Repository)
CentralRepoIngestModule_prevSeenOsAcctSetName=Users seen in previous cases
CentralRepoIngestModule_prevSeenSetName=Previously Seen (Central Repository)
CentralRepoIngestModule_prevUnseenJustification=Previously seen in zero cases
CentralRepoIngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation CentralRepoIngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation
CentralRepoIngestModuleFactory.ingestmodule.name=Central Repository CentralRepoIngestModuleFactory.ingestmodule.name=Central Repository
CrDataArtifactIngestModule_crInaccessibleErrMsg=Error accessing central repository
CrDataArtifactIngestModule_crNotEnabledErrMsg=Central repository required, but not enabled
CrDataArtifactIngestModule_noCurrentCaseErrMsg=Error getting current case
# {0} - list of cases
CrDataArtifactIngestModule_notableJustification=Previously marked as notable in cases {0}
CrDataArtifactIngestModule_notableSetName=Previously Tagged As Notable (Central Repository)
CrDataArtifactIngestModule_osAcctMgrInaccessibleErrMsg=Error getting OS accounts manager
# {0} - list of cases
CrDataArtifactIngestModule_prevSeenJustification=Previously seen in cases {0}
CrDataArtifactIngestModule_prevSeenOsAcctConfig=Previously Seen Users (Central Repository)
CrDataArtifactIngestModule_prevSeenOsAcctSetName=Users seen in previous cases
CrDataArtifactIngestModule_prevSeenSetName=Previously Seen (Central Repository)
CrDataArtifactIngestModule_prevUnseenJustification=Previously seen in zero cases
IngestSettingsPanel.ingestSettingsLabel.text=Ingest Settings IngestSettingsPanel.ingestSettingsLabel.text=Ingest Settings
IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable
IngestSettingsPanel.flagPreviouslySeenDevicesCheckbox.text=Flag devices and users previously seen in other cases IngestSettingsPanel.flagPreviouslySeenDevicesCheckbox.text=Flag devices and users previously seen in other cases

59
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java
View File

@ -33,7 +33,9 @@ import org.apache.commons.lang3.StringUtils;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoDbManager;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoPlatforms;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
@ -103,16 +105,24 @@ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestMo
} }
@NbBundle.Messages({ @NbBundle.Messages({
"CrDataArtifactIngestModule_crNotEnabledErrMsg=Central repository required, but not enabled", "CentralRepoIngestModule_crNotEnabledErrMsg=Central repository required, but not enabled",
"CrDataArtifactIngestModule_noCurrentCaseErrMsg=Error getting current case", "CentralRepoIngestModule_noCurrentCaseErrMsg=Error getting current case",
"CrDataArtifactIngestModule_osAcctMgrInaccessibleErrMsg=Error getting OS accounts manager", "CentralRepoIngestModule_osAcctMgrInaccessibleErrMsg=Error getting OS accounts manager",
"CrDataArtifactIngestModule_crInaccessibleErrMsg=Error accessing central repository",}) "CentralRepoIngestModule_crInaccessibleErrMsg=Error accessing central repository",
"CentralRepoIngestModule_crDatabaseTypeMismatch=Mulit-user cases require a PostgreSQL central repository"
})
@Override @Override
public void startUp(IngestJobContext context) throws IngestModuleException { public void startUp(IngestJobContext context) throws IngestModuleException {
/*
* IMPORTANT: Start up IngestModuleException messages are displayed to
* the user, if a user is present. Therefore, an exception to the policy
* that exception messages are not localized is appropriate here. Also,
* the exception messages should be user-friendly.
*/
dataSource = context.getDataSource(); dataSource = context.getDataSource();
ingestJobId = context.getJobId(); ingestJobId = context.getJobId();
if (!CentralRepository.isEnabled()) { if (!CentralRepository.isEnabled()) {
throw new IngestModuleException(Bundle.CrDataArtifactIngestModule_crNotEnabledErrMsg()); // May be displayed to user. throw new IngestModuleException(Bundle.CentralRepoIngestModule_crNotEnabledErrMsg()); // May be displayed to user.
} }
try { try {
currentCase = Case.getCurrentCaseThrows(); currentCase = Case.getCurrentCaseThrows();
@ -121,12 +131,17 @@ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestMo
osAccountMgr = tskCase.getOsAccountManager(); osAccountMgr = tskCase.getOsAccountManager();
centralRepo = CentralRepository.getInstance(); centralRepo = CentralRepository.getInstance();
} catch (NoCurrentCaseException ex) { } catch (NoCurrentCaseException ex) {
throw new IngestModuleException(Bundle.CrDataArtifactIngestModule_noCurrentCaseErrMsg(), ex); // May be displayed to user. throw new IngestModuleException(Bundle.CentralRepoIngestModule_noCurrentCaseErrMsg(), ex);
} catch (TskCoreException ex) { } catch (TskCoreException ex) {
throw new IngestModuleException(Bundle.CrDataArtifactIngestModule_osAcctMgrInaccessibleErrMsg(), ex); // May be displayed to user. throw new IngestModuleException(Bundle.CentralRepoIngestModule_osAcctMgrInaccessibleErrMsg(), ex);
} catch (CentralRepoException ex) { } catch (CentralRepoException ex) {
throw new IngestModuleException(Bundle.CrDataArtifactIngestModule_crInaccessibleErrMsg(), ex); // May be displayed to user. throw new IngestModuleException(Bundle.CentralRepoIngestModule_crInaccessibleErrMsg(), ex);
} }
// Don't allow sqlite central repo databases to be used for multi user cases
if ((currentCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) && (CentralRepoDbManager.getSavedDbChoice().getDbPlatform() == CentralRepoPlatforms.SQLITE)) {
throw new IngestModuleException(Bundle.CentralRepoIngestModule_crDatabaseTypeMismatch());
}
} }
/** /**
@ -187,8 +202,8 @@ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestMo
* accounts if they have been seen in other cases. * accounts if they have been seen in other cases.
*/ */
@NbBundle.Messages({ @NbBundle.Messages({
"CrDataArtifactIngestModule_prevSeenOsAcctSetName=Users seen in previous cases", "CentralRepoIngestModule_prevSeenOsAcctSetName=Users seen in previous cases",
"CrDataArtifactIngestModule_prevSeenOsAcctConfig=Previously Seen Users (Central Repository)" "CentralRepoIngestModule_prevSeenOsAcctConfig=Previously Seen Users (Central Repository)"
}) })
private void analyzeOsAccounts() { private void analyzeOsAccounts() {
try { try {
@ -360,14 +375,14 @@ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestMo
* @param corrAttrValue The value of the matched correlation attribute. * @param corrAttrValue The value of the matched correlation attribute.
*/ */
@NbBundle.Messages({ @NbBundle.Messages({
"CrDataArtifactIngestModule_notableSetName=Previously Tagged As Notable (Central Repository)", "CentralRepoIngestModule_notableSetName=Previously Tagged As Notable (Central Repository)",
"# {0} - list of cases", "# {0} - list of cases",
"CrDataArtifactIngestModule_notableJustification=Previously marked as notable in cases {0}" "CentralRepoIngestModule_notableJustification=Previously marked as notable in cases {0}"
}) })
private void makePrevNotableAnalysisResult(Content content, Set<String> previousCases, CorrelationAttributeInstance.Type corrAttrType, String corrAttrValue) { private void makePrevNotableAnalysisResult(Content content, Set<String> previousCases, CorrelationAttributeInstance.Type corrAttrType, String corrAttrValue) {
String prevCases = previousCases.stream().collect(Collectors.joining(",")); String prevCases = previousCases.stream().collect(Collectors.joining(","));
String justification = Bundle.CrDataArtifactIngestModule_notableJustification(prevCases); String justification = Bundle.CentralRepoIngestModule_notableJustification(prevCases);
Collection<BlackboardAttribute> attributes = Arrays.asList(new BlackboardAttribute(TSK_SET_NAME, CentralRepoIngestModuleFactory.getModuleName(), Bundle.CrDataArtifactIngestModule_notableSetName()), Collection<BlackboardAttribute> attributes = Arrays.asList(new BlackboardAttribute(TSK_SET_NAME, CentralRepoIngestModuleFactory.getModuleName(), Bundle.CentralRepoIngestModule_notableSetName()),
new BlackboardAttribute(TSK_CORRELATION_TYPE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrType.getDisplayName()), new BlackboardAttribute(TSK_CORRELATION_TYPE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrType.getDisplayName()),
new BlackboardAttribute(TSK_CORRELATION_VALUE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrValue), new BlackboardAttribute(TSK_CORRELATION_VALUE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrValue),
new BlackboardAttribute(TSK_OTHER_CASES, CentralRepoIngestModuleFactory.getModuleName(), prevCases)); new BlackboardAttribute(TSK_OTHER_CASES, CentralRepoIngestModuleFactory.getModuleName(), prevCases));
@ -385,17 +400,17 @@ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestMo
* @param corrAttrValue The value of the matched correlation attribute. * @param corrAttrValue The value of the matched correlation attribute.
*/ */
@NbBundle.Messages({ @NbBundle.Messages({
"CrDataArtifactIngestModule_prevSeenSetName=Previously Seen (Central Repository)", "CentralRepoIngestModule_prevSeenSetName=Previously Seen (Central Repository)",
"# {0} - list of cases", "# {0} - list of cases",
"CrDataArtifactIngestModule_prevSeenJustification=Previously seen in cases {0}" "CentralRepoIngestModule_prevSeenJustification=Previously seen in cases {0}"
}) })
private void makePrevSeenAnalysisResult(Content content, Set<String> previousCases, CorrelationAttributeInstance.Type corrAttrType, String corrAttrValue) { private void makePrevSeenAnalysisResult(Content content, Set<String> previousCases, CorrelationAttributeInstance.Type corrAttrType, String corrAttrValue) {
Optional<Score> score = calculateScore(previousCases.size()); Optional<Score> score = calculateScore(previousCases.size());
if (score.isPresent()) { if (score.isPresent()) {
String prevCases = previousCases.stream().collect(Collectors.joining(",")); String prevCases = previousCases.stream().collect(Collectors.joining(","));
String justification = Bundle.CrDataArtifactIngestModule_prevSeenJustification(prevCases); String justification = Bundle.CentralRepoIngestModule_prevSeenJustification(prevCases);
Collection<BlackboardAttribute> analysisResultAttributes = Arrays.asList( Collection<BlackboardAttribute> analysisResultAttributes = Arrays.asList(
new BlackboardAttribute(TSK_SET_NAME, CentralRepoIngestModuleFactory.getModuleName(), Bundle.CrDataArtifactIngestModule_prevSeenSetName()), new BlackboardAttribute(TSK_SET_NAME, CentralRepoIngestModuleFactory.getModuleName(), Bundle.CentralRepoIngestModule_prevSeenSetName()),
new BlackboardAttribute(TSK_CORRELATION_TYPE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrType.getDisplayName()), new BlackboardAttribute(TSK_CORRELATION_TYPE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrType.getDisplayName()),
new BlackboardAttribute(TSK_CORRELATION_VALUE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrValue), new BlackboardAttribute(TSK_CORRELATION_VALUE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrValue),
new BlackboardAttribute(TSK_OTHER_CASES, CentralRepoIngestModuleFactory.getModuleName(), prevCases)); new BlackboardAttribute(TSK_OTHER_CASES, CentralRepoIngestModuleFactory.getModuleName(), prevCases));
@ -411,13 +426,13 @@ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestMo
* @param corrAttrValue The value of the new correlation attribute. * @param corrAttrValue The value of the new correlation attribute.
*/ */
@NbBundle.Messages({ @NbBundle.Messages({
"CrDataArtifactIngestModule_prevUnseenJustification=Previously seen in zero cases" "CentralRepoIngestModule_prevUnseenJustification=Previously seen in zero cases"
}) })
private void makePrevUnseenAnalysisResult(Content content, CorrelationAttributeInstance.Type corrAttrType, String corrAttrValue) { private void makePrevUnseenAnalysisResult(Content content, CorrelationAttributeInstance.Type corrAttrType, String corrAttrValue) {
Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList( Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(
new BlackboardAttribute(TSK_CORRELATION_TYPE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrType.getDisplayName()), new BlackboardAttribute(TSK_CORRELATION_TYPE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrType.getDisplayName()),
new BlackboardAttribute(TSK_CORRELATION_VALUE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrValue)); new BlackboardAttribute(TSK_CORRELATION_VALUE, CentralRepoIngestModuleFactory.getModuleName(), corrAttrValue));
makeAndPostAnalysisResult(content, BlackboardArtifact.Type.TSK_PREVIOUSLY_UNSEEN, attributesForNewArtifact, "", Score.SCORE_LIKELY_NOTABLE, Bundle.CrDataArtifactIngestModule_prevUnseenJustification()); makeAndPostAnalysisResult(content, BlackboardArtifact.Type.TSK_PREVIOUSLY_UNSEEN, attributesForNewArtifact, "", Score.SCORE_LIKELY_NOTABLE, Bundle.CentralRepoIngestModule_prevUnseenJustification());
} }
/** /**
@ -437,8 +452,8 @@ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestMo
score = Score.SCORE_NONE; score = Score.SCORE_NONE;
} }
return Optional.ofNullable(score); return Optional.ofNullable(score);
} }
/** /**
* Makes a new analysis result of a given type for a content and posts it to * Makes a new analysis result of a given type for a content and posts it to
* the blackboard. * the blackboard.

176
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoFileIngestModule.java
View File

@ -32,11 +32,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeUtil; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeUtil;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoPlatforms;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoDbManager;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.healthmonitor.HealthMonitor; import org.sleuthkit.autopsy.healthmonitor.HealthMonitor;
import org.sleuthkit.autopsy.healthmonitor.TimingMetric; import org.sleuthkit.autopsy.healthmonitor.TimingMetric;
import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.FileIngestModule;
@ -59,18 +55,18 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository;
import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.Score;
/** /**
* Ingest module for inserting entries into the Central Repository database on * A file ingest module that adds correlation attributes for files to the
* ingest of a data source * central repository and makes previously notable analysis results based on
* previous occurences.
*/ */
@Messages({"CentralRepoIngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)", @Messages({"CentralRepoIngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
"CentralRepoIngestModule.prevCaseComment.text=Previous Case: "}) "CentralRepoIngestModule.prevCaseComment.text=Previous Case: "})
final class CentralRepoFileIngestModule implements FileIngestModule { final class CentralRepoFileIngestModule implements FileIngestModule {
private final static Logger logger = Logger.getLogger(CentralRepoFileIngestModule.class.getName()); private static final Logger logger = Logger.getLogger(CentralRepoFileIngestModule.class.getName());
private static final String MODULE_NAME = CentralRepoIngestModuleFactory.getModuleName(); private static final String MODULE_NAME = CentralRepoIngestModuleFactory.getModuleName();
private final IngestServices services = IngestServices.getInstance(); private final IngestServices services = IngestServices.getInstance();
private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter();
private long jobId; private long jobId;
private CorrelationCase centralRepoCase; private CorrelationCase centralRepoCase;
private CorrelationDataSource centralRepoDataSource; private CorrelationDataSource centralRepoDataSource;
@ -78,11 +74,14 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
private final boolean flagTaggedNotableItems; private final boolean flagTaggedNotableItems;
private Blackboard blackboard; private Blackboard blackboard;
private final boolean createCorrelationProperties; private final boolean createCorrelationProperties;
private CentralRepository centralRepoDb;
/** /**
* Instantiate the Central Repository ingest module. * Constructs a file ingest module that adds correlation attributes for
* files to the central repository and makes previously notable analysis
* results based on previous occurences.
* *
* @param settings The ingest settings for the module instance. * @param settings The ingest job settings.
*/ */
CentralRepoFileIngestModule(IngestSettings settings) { CentralRepoFileIngestModule(IngestSettings settings) {
flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); flagTaggedNotableItems = settings.isFlagTaggedNotableItems();
@ -91,22 +90,6 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
@Override @Override
public ProcessResult process(AbstractFile abstractFile) { public ProcessResult process(AbstractFile abstractFile) {
if (CentralRepository.isEnabled() == false) {
/*
* Not signaling an error for now. This is a workaround for the way
* all newly didscovered ingest modules are automatically anabled.
*
* TODO (JIRA-2731): Add isEnabled API for ingest modules.
*/
return ProcessResult.OK;
}
try {
blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
} catch (NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Exception while getting open case.", ex);
return ProcessResult.ERROR;
}
if (!CorrelationAttributeUtil.isSupportedAbstractFileType(abstractFile)) { if (!CorrelationAttributeUtil.isSupportedAbstractFileType(abstractFile)) {
return ProcessResult.OK; return ProcessResult.OK;
@ -116,15 +99,6 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
return ProcessResult.OK; return ProcessResult.OK;
} }
CentralRepository dbManager;
try {
dbManager = CentralRepository.getInstance();
} catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
return ProcessResult.ERROR;
}
// only continue if we are correlating filesType
if (!filesType.isEnabled()) { if (!filesType.isEnabled()) {
return ProcessResult.OK; return ProcessResult.OK;
} }
@ -142,7 +116,7 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) { if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) {
try { try {
TimingMetric timingMetric = HealthMonitor.getTimingMetric("Central Repository: Notable artifact query"); TimingMetric timingMetric = HealthMonitor.getTimingMetric("Central Repository: Notable artifact query");
List<String> caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); List<String> caseDisplayNamesList = centralRepoDb.getListCasesHavingArtifactInstancesKnownBad(filesType, md5);
HealthMonitor.submitTimingMetric(timingMetric); HealthMonitor.submitTimingMetric(timingMetric);
if (!caseDisplayNamesList.isEmpty()) { if (!caseDisplayNamesList.isEmpty()) {
postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList, filesType, md5); postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList, filesType, md5);
@ -169,7 +143,7 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database. TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database.
, ,
abstractFile.getId()); abstractFile.getId());
dbManager.addAttributeInstanceBulk(cefi); centralRepoDb.addAttributeInstanceBulk(cefi);
} catch (CentralRepoException ex) { } catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
return ProcessResult.ERROR; return ProcessResult.ERROR;
@ -183,113 +157,68 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
@Override @Override
public void shutDown() { public void shutDown() {
if ((CentralRepository.isEnabled() == false) || (centralRepoCase == null) || (centralRepoDataSource == null)) { if (refCounter.decrementAndGet(jobId) == 0) {
return; try {
centralRepoDb.commitAttributeInstancesBulk();
} catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error committing bulk insert of correlation attributes", ex); // NON-NLS
}
} }
CentralRepository dbManager;
try {
dbManager = CentralRepository.getInstance();
} catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex);
return;
}
try {
dbManager.commitAttributeInstancesBulk();
} catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS
}
try {
Long count = dbManager.getCountArtifactInstancesByCaseDataSource(centralRepoDataSource);
logger.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, centralRepoCase.getDisplayName(), centralRepoDataSource.getName()}); // NON-NLS
} catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS
}
// TODO: once we implement shared cache, if refCounter is 1, then submit data in bulk.
refCounter.decrementAndGet(jobId);
} }
// see ArtifactManagerTimeTester for details
@Messages({ @Messages({
"CentralRepoIngestModule.notfyBubble.title=Central Repository Not Initialized", "CentralRepoIngestModule_missingFileCorrAttrTypeErrMsg=Correlation attribute type for files not found in the central repository",
"CentralRepoIngestModule.errorMessage.isNotEnabled=Central repository settings are not initialized, cannot run Central Repository ingest module." "CentralRepoIngestModule_cannotGetCrCaseErrMsg=Case not present in the central repository",
"CentralRepoIngestModule_cannotGetCrDataSourceErrMsg=Data source not present in the central repository"
}) })
@Override @Override
public void startUp(IngestJobContext context) throws IngestModuleException { public void startUp(IngestJobContext context) throws IngestModuleException {
if (CentralRepository.isEnabled() == false) { jobId = context.getJobId();
/*
* Not throwing the customary exception for now. This is a /*
* workaround for the way all newly didscovered ingest modules are * IMPORTANT: Start up IngestModuleException messages are displayed to
* automatically anabled. * the user, if a user is present. Therefore, an exception to the policy
* * that exception messages are not localized is appropriate here. Also,
* TODO (JIRA-2731): Add isEnabled API for ingest modules. * the exception messages should be user-friendly.
*/ */
if (RuntimeProperties.runningWithGUI()) { if (!CentralRepository.isEnabled()) {
if (1L == warningMsgRefCounter.incrementAndGet(jobId)) { throw new IngestModuleException(Bundle.CrDataArtifactIngestModule_crNotEnabledErrMsg());
MessageNotifyUtil.Notify.warn(Bundle.CentralRepoIngestModule_notfyBubble_title(), Bundle.CentralRepoIngestModule_errorMessage_isNotEnabled());
}
}
return;
} }
Case autopsyCase; Case autopsyCase;
try { try {
autopsyCase = Case.getCurrentCaseThrows(); autopsyCase = Case.getCurrentCaseThrows();
} catch (NoCurrentCaseException ex) { } catch (NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Exception while getting open case.", ex); throw new IngestModuleException(Bundle.CrDataArtifactIngestModule_noCurrentCaseErrMsg(), ex);
throw new IngestModuleException("Exception while getting open case.", ex);
} }
// Don't allow sqlite central repo databases to be used for multi user cases blackboard = autopsyCase.getSleuthkitCase().getBlackboard();
if ((autopsyCase.getCaseType() == Case.CaseType.MULTI_USER_CASE)
&& (CentralRepoDbManager.getSavedDbChoice().getDbPlatform() == CentralRepoPlatforms.SQLITE)) {
logger.log(Level.SEVERE, "Cannot run Central Repository ingest module on a multi-user case with a SQLite central repository.");
throw new IngestModuleException("Cannot run on a multi-user case with a SQLite central repository."); // NON-NLS
}
jobId = context.getJobId();
CentralRepository centralRepoDb;
try { try {
centralRepoDb = CentralRepository.getInstance(); centralRepoDb = CentralRepository.getInstance();
} catch (CentralRepoException ex) { } catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS throw new IngestModuleException(Bundle.CentralRepoIngestModule_crInaccessibleErrMsg(), ex);
throw new IngestModuleException("Error connecting to central repository database.", ex); // NON-NLS
} }
try { try {
filesType = centralRepoDb.getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID); filesType = centralRepoDb.getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID);
} catch (CentralRepoException ex) { } catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS throw new IngestModuleException(Bundle.CentralRepoIngestModule_missingFileCorrAttrTypeErrMsg(), ex);
throw new IngestModuleException("Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS
} }
try { try {
centralRepoCase = centralRepoDb.getCase(autopsyCase); centralRepoCase = centralRepoDb.getCase(autopsyCase);
} catch (CentralRepoException ex) { } catch (CentralRepoException ex) {
throw new IngestModuleException("Unable to get case from central repository database ", ex); throw new IngestModuleException(Bundle.CentralRepoIngestModule_cannotGetCrCaseErrMsg(), ex);
} }
try { try {
centralRepoDataSource = CorrelationDataSource.fromTSKDataSource(centralRepoCase, context.getDataSource()); centralRepoDataSource = CorrelationDataSource.fromTSKDataSource(centralRepoCase, context.getDataSource());
} catch (CentralRepoException ex) { } catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS throw new IngestModuleException(Bundle.CentralRepoIngestModule_cannotGetCrDataSourceErrMsg(), ex);
throw new IngestModuleException("Error getting data source info.", ex); // NON-NLS
} }
// TODO: once we implement a shared cache, load/init it here w/ syncronized and define reference counter
// if we are the first thread / module for this job, then make sure the case
// and image exist in the DB before we associate artifacts with it.
if (refCounter.incrementAndGet(jobId)
== 1) {
// ensure we have this data source in the EAM DB
try {
if (null == centralRepoDb.getDataSource(centralRepoCase, centralRepoDataSource.getDataSourceObjectID())) {
centralRepoDb.newDataSource(centralRepoDataSource);
}
} catch (CentralRepoException ex) {
logger.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS
throw new IngestModuleException("Error adding data source to Central Repository.", ex); // NON-NLS
}
} refCounter.incrementAndGet(jobId);
} }
/** /**
@ -298,19 +227,18 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
* @param abstractFile The file from which to create an artifact. * @param abstractFile The file from which to create an artifact.
* @param caseDisplayNames Case names to be added to a TSK_COMMON attribute. * @param caseDisplayNames Case names to be added to a TSK_COMMON attribute.
*/ */
private void postCorrelatedBadFileToBlackboard(AbstractFile abstractFile, List<String> caseDisplayNames, CorrelationAttributeInstance.Type aType, String value) { private void postCorrelatedBadFileToBlackboard(AbstractFile abstractFile, List<String> caseDisplayNames, CorrelationAttributeInstance.Type corrAtrrType, String corrAttrValue) {
String prevCases = caseDisplayNames.stream().distinct().collect(Collectors.joining(",")); String prevCases = caseDisplayNames.stream().distinct().collect(Collectors.joining(","));
String justification = "Previously marked as notable in cases " + prevCases; String justification = "Previously marked as notable in cases " + prevCases;
Collection<BlackboardAttribute> attributes = Arrays.asList( Collection<BlackboardAttribute> attributes = Arrays.asList(new BlackboardAttribute(
new BlackboardAttribute( TSK_SET_NAME, MODULE_NAME,
TSK_SET_NAME, MODULE_NAME, Bundle.CentralRepoIngestModule_prevTaggedSet_text()),
Bundle.CentralRepoIngestModule_prevTaggedSet_text()),
new BlackboardAttribute( new BlackboardAttribute(
TSK_CORRELATION_TYPE, MODULE_NAME, TSK_CORRELATION_TYPE, MODULE_NAME,
aType.getDisplayName()), corrAtrrType.getDisplayName()),
new BlackboardAttribute( new BlackboardAttribute(
TSK_CORRELATION_VALUE, MODULE_NAME, TSK_CORRELATION_VALUE, MODULE_NAME,
value), corrAttrValue),
new BlackboardAttribute( new BlackboardAttribute(
TSK_OTHER_CASES, MODULE_NAME, TSK_OTHER_CASES, MODULE_NAME,
prevCases)); prevCases));
@ -336,14 +264,6 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
} }
} }
@Messages({
"CentralRepoIngestModule_notable_message_header=<html>A file in this data source was previously seen and tagged as Notable.<br>",
"CentralRepoIngestModel_name_header=Name:<br>",
"CentralRepoIngestModel_previous_case_header=<br>Previous Cases:<br>",
"# {0} - Name of file that is Notable",
"CentralRepoIngestModule_postToBB_knownBadMsg=Notable: {0}"
})
/** /**
* Post a message to the ingest inbox alerting the user that a bad file was * Post a message to the ingest inbox alerting the user that a bad file was
* found. * found.
@ -353,6 +273,13 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
* @param md5Hash badFile's md5 hash * @param md5Hash badFile's md5 hash
* @param caseDisplayNames List of cases that the artifact appears in. * @param caseDisplayNames List of cases that the artifact appears in.
*/ */
@Messages({
"CentralRepoIngestModule_notable_message_header=<html>A file in this data source was previously seen and tagged as Notable.<br>",
"CentralRepoIngestModel_name_header=Name:<br>",
"CentralRepoIngestModel_previous_case_header=<br>Previous Cases:<br>",
"# {0} - Name of file that is Notable",
"CentralRepoIngestModule_postToBB_knownBadMsg=Notable: {0}"
})
private void sendBadFileInboxMessage(BlackboardArtifact artifact, String name, String md5Hash, List<String> caseDisplayNames) { private void sendBadFileInboxMessage(BlackboardArtifact artifact, String name, String md5Hash, List<String> caseDisplayNames) {
StringBuilder detailsSb = new StringBuilder(1024); StringBuilder detailsSb = new StringBuilder(1024);
@ -368,4 +295,5 @@ final class CentralRepoFileIngestModule implements FileIngestModule {
name + md5Hash, name + md5Hash,
artifact)); artifact));
} }
} }