still need to work on portable case and user custom events

This commit is contained in:
Greg DiCristofaro 2021-04-26 14:31:02 -04:00
parent 217da17a18
commit 6fb71a03e8
3 changed files with 248 additions and 183 deletions

View File

@ -28,6 +28,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
@ -56,10 +57,50 @@ import org.sleuthkit.datamodel.TskCoreException;
/**
* A runnable that - copy the logical image folder to a destination folder - add
* SearchResults.txt and *_users.txt files to report - add an image data source to the
* case database.
* SearchResults.txt and *_users.txt files to report - add an image data source
* to the case database.
*/
final class AddLogicalImageTask implements Runnable {
/**
* Information about a file including the object id of the file as well as
* the object id of the data source.
*/
private static class FileId {
private final long dataSourceId;
private final long fileId;
/**
* Main constructor.
*
* @param dataSourceId Object Id of the data source.
* @param fileId Object Id of the file.
*/
FileId(long dataSourceId, long fileId) {
this.dataSourceId = dataSourceId;
this.fileId = fileId;
}
/**
* Returns the data source id of the file.
*
* @return The data source id of the file.
*/
long getDataSourceId() {
return dataSourceId;
}
/**
* Returns the object id of the file.
*
* @return The object id of the file.
*/
long getFileId() {
return fileId;
}
}
private final static BlackboardArtifact.Type INTERESTING_FILE_TYPE = new BlackboardArtifact.Type(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT);
private final static Logger LOGGER = Logger.getLogger(AddLogicalImageTask.class.getName());
@ -108,8 +149,8 @@ final class AddLogicalImageTask implements Runnable {
}
/**
* Add SearchResults.txt and *_users.txt to the case
* report Adds the image to the case database.
* Add SearchResults.txt and *_users.txt to the case report Adds the image
* to the case database.
*/
@Messages({
"# {0} - src", "# {1} - dest", "AddLogicalImageTask.copyingImageFromTo=Copying image from {0} to {1}",
@ -217,7 +258,7 @@ final class AddLogicalImageTask implements Runnable {
}
List<Content> newDataSources = new ArrayList<>();
Map<String, List<Long>> interestingFileMap = new HashMap<>();
Map<String, List<FileId>> interestingFileMap = new HashMap<>();
if (imagePaths.isEmpty()) {
createVHD = false;
@ -361,11 +402,11 @@ final class AddLogicalImageTask implements Runnable {
"# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingInterestingFile=Adding interesting files ({0}/{1})",
"AddLogicalImageTask.logicalImagerResults=Logical Imager results"
})
private void addInterestingFiles(Map<String, List<Long>> interestingFileMap) throws IOException, TskCoreException {
private void addInterestingFiles(Map<String, List<FileId>> interestingFileMap) throws IOException, TskCoreException {
int lineNumber = 0;
List<BlackboardArtifact> artifacts = new ArrayList<>();
Iterator<Map.Entry<String, List<Long>>> iterator = interestingFileMap.entrySet().iterator();
Iterator<Map.Entry<String, List<FileId>>> iterator = interestingFileMap.entrySet().iterator();
while (iterator.hasNext()) {
if (cancelled) {
@ -374,14 +415,14 @@ final class AddLogicalImageTask implements Runnable {
break;
}
Map.Entry<String, List<Long>> entry = iterator.next();
Map.Entry<String, List<FileId>> entry = iterator.next();
String key = entry.getKey();
String ruleName;
String[] split = key.split("\t");
ruleName = split[1];
List<Long> fileIds = entry.getValue();
for (Long fileId: fileIds) {
List<FileId> fileIds = entry.getValue();
for (FileId fileId : fileIds) {
if (cancelled) {
postArtifacts(artifacts);
return;
@ -393,7 +434,7 @@ final class AddLogicalImageTask implements Runnable {
postArtifacts(artifacts);
artifacts.clear();
}
addInterestingFileToArtifacts(fileId, Bundle.AddLogicalImageTask_logicalImagerResults(), ruleName, artifacts);
addInterestingFileToArtifacts(fileId.getFileId(), fileId.getDataSourceId(), Bundle.AddLogicalImageTask_logicalImagerResults(), ruleName, artifacts);
lineNumber++;
}
iterator.remove();
@ -401,31 +442,30 @@ final class AddLogicalImageTask implements Runnable {
postArtifacts(artifacts);
}
private void addInterestingFileToArtifacts(long fileId, String ruleSetName, String ruleName, List<BlackboardArtifact> artifacts) throws TskCoreException {
switch (INTERESTING_FILE_TYPE.getCategory()) {
case DATA_ARTIFACT:
case ANALYSIS_RESULT:
return this.currentCase.getSleuthkitCase().getBlackboard().newAnalysisResult(INTERESTING_FILE_TYPE, fileId, fileId, Score.SCORE_UNKNOWN, null, null, null);
default:
throw new TskCoreException("Unknown category: " + INTERESTING_FILE_TYPE.getCategory().getDisplayName());
}
Collection<BlackboardAttribute> attributes = new ArrayList<>();
BlackboardAttribute setNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, ruleSetName);
attributes.add(setNameAttribute);
BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY, MODULE_NAME, ruleName);
attributes.add(ruleNameAttribute);
//BlackboardArtifact artifact = this.currentCase.getSleuthkitCase().newBlackboardArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, fileId);
artifact.addAttributes(attributes);
private void addInterestingFileToArtifacts(long fileId, long dataSourceId, String ruleSetName, String ruleName, List<BlackboardArtifact> artifacts) throws TskCoreException {
BlackboardArtifact artifact = this.blackboard.newAnalysisResult(
INTERESTING_FILE_TYPE,
fileId,
dataSourceId,
Score.SCORE_UNKNOWN,
null,
null,
null,
Arrays.asList(
new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, ruleSetName),
new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY, MODULE_NAME, ruleName)
));
artifacts.add(artifact);
}
@Messages({
"# {0} - file number", "# {1} - total files", "AddLogicalImageTask.searchingInterestingFile=Searching for interesting files ({0}/{1})"
})
private Map<String, List<Long>> getInterestingFileMapForVHD(Path resultsPath) throws TskCoreException, IOException {
private Map<String, List<FileId>> getInterestingFileMapForVHD(Path resultsPath) throws TskCoreException, IOException {
Map<Long, List<String>> objIdToimagePathsMap = currentCase.getSleuthkitCase().getImagePaths();
imagePathToObjIdMap = imagePathsToDataSourceObjId(objIdToimagePathsMap);
Map<String, List<Long>> interestingFileMap = new HashMap<>();
Map<String, List<FileId>> interestingFileMap = new HashMap<>();
try (BufferedReader br = new BufferedReader(new InputStreamReader(
new FileInputStream(resultsPath.toFile()), "UTF8"))) { // NON-NLS
@ -458,16 +498,14 @@ final class AddLogicalImageTask implements Runnable {
String query = makeQuery(vhdFilename, fileMetaAddressStr, parentPath, filename);
List<AbstractFile> matchedFiles = Case.getCurrentCase().getSleuthkitCase().findAllFilesWhere(query);
List<Long> fileIds = new ArrayList<>();
List<FileId> fileIds = new ArrayList<>();
for (AbstractFile file : matchedFiles) {
fileIds.add(file.getId());
fileIds.add(new FileId(file.getDataSourceObjectId(), file.getId()));
}
String key = String.format("%s\t%s", ruleSetName, ruleName);
if (interestingFileMap.containsKey(key)) {
interestingFileMap.get(key).addAll(fileIds);
} else {
interestingFileMap.put(key, fileIds);
}
interestingFileMap.computeIfAbsent(key, (k) -> new ArrayList<>())
.addAll(fileIds);
lineNumber++;
} // end reading file
}
@ -486,10 +524,10 @@ final class AddLogicalImageTask implements Runnable {
@Messages({
"# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingExtractedFile=Adding extracted files ({0}/{1})"
})
private Map<String, List<Long>> addExtractedFiles(File src, Path resultsPath, Host host, List<Content> newDataSources) throws TskCoreException, IOException {
private Map<String, List<FileId>> addExtractedFiles(File src, Path resultsPath, Host host, List<Content> newDataSources) throws TskCoreException, IOException {
SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase();
SleuthkitCase.CaseDbTransaction trans = null;
Map<String, List<Long>> interestingFileMap = new HashMap<>();
Map<String, List<FileId>> interestingFileMap = new HashMap<>();
try {
trans = skCase.beginTransaction();
@ -542,12 +580,11 @@ final class AddLogicalImageTask implements Runnable {
Long.parseLong(mtime),
localFilesDataSource);
String key = String.format("%s\t%s", ruleSetName, ruleName);
List<Long> value = new ArrayList<>();
if (interestingFileMap.containsKey(key)) {
value = interestingFileMap.get(key);
}
value.add(fileAdded.getId());
interestingFileMap.put(key, value);
long dataSourceId = fileAdded.getDataSourceObjectId();
long fileId = fileAdded.getId();
interestingFileMap.computeIfAbsent(key, (k) -> new ArrayList<>())
.add(new FileId(dataSourceId, fileId));
lineNumber++;
} // end reading file
}

View File

@ -40,13 +40,14 @@ import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Score;
import org.sleuthkit.datamodel.TskDataException;
/**
* Data source ingest module that verifies the integrity of an Expert Witness
* Format (EWF) E01 image file by generating a hash of the file and comparing it
* to the value stored in the image. Will also generate hashes for any image-type
* data source that has none.
* to the value stored in the image. Will also generate hashes for any
* image-type data source that has none.
*/
public class DataSourceIntegrityIngestModule implements DataSourceIngestModule {
@ -110,8 +111,7 @@ public class DataSourceIntegrityIngestModule implements DataSourceIngestModule {
"# {0} - imageName",
"DataSourceIntegrityIngestModule.process.verificationSuccess=Integrity of {0} verified",
"# {0} - imageName",
"DataSourceIntegrityIngestModule.process.verificationFailure={0} failed integrity verification",
})
"DataSourceIntegrityIngestModule.process.verificationFailure={0} failed integrity verification",})
@Override
public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
String imgName = dataSource.getName();
@ -136,7 +136,6 @@ public class DataSourceIntegrityIngestModule implements DataSourceIngestModule {
// Determine which mode we're in.
// - If there are any preset hashes, then we'll verify them (assuming the verify checkbox is selected)
// - Otherwise we'll calculate and store all three hashes (assuming the compute checkbox is selected)
// First get a list of all stored hash types
try {
if (img.getMd5() != null && !img.getMd5().isEmpty()) {
@ -294,10 +293,16 @@ public class DataSourceIntegrityIngestModule implements DataSourceIngestModule {
if (!verified) {
try {
BlackboardArtifact verificationFailedArtifact = Case.getCurrentCase().getSleuthkitCase().newBlackboardArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_VERIFICATION_FAILED, img.getId());
verificationFailedArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT,
DataSourceIntegrityModuleFactory.getModuleName(), artifactComment));
Case.getCurrentCase().getServices().getArtifactsBlackboard().postArtifact(verificationFailedArtifact, DataSourceIntegrityModuleFactory.getModuleName());
BlackboardArtifact verificationFailedArtifact = Case.getCurrentCase().getSleuthkitCase().getBlackboard().newAnalysisResult(
new BlackboardArtifact.Type(BlackboardArtifact.ARTIFACT_TYPE.TSK_VERIFICATION_FAILED),
img.getId(), img.getId(),
Score.SCORE_UNKNOWN,
null, null, null,
Arrays.asList(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT,
DataSourceIntegrityModuleFactory.getModuleName(), artifactComment)));
Case.getCurrentCase().getServices().getArtifactsBlackboard()
.postArtifact(verificationFailedArtifact, DataSourceIntegrityModuleFactory.getModuleName());
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error creating verification failed artifact", ex);
} catch (Blackboard.BlackboardException ex) {
@ -366,8 +371,8 @@ public class DataSourceIntegrityIngestModule implements DataSourceIngestModule {
}
/**
* Enum to hold the type of hash.
* The value in the "name" field should be compatible with MessageDigest
* Enum to hold the type of hash. The value in the "name" field should be
* compatible with MessageDigest
*/
private enum HashType {
MD5("MD5"),
@ -389,6 +394,7 @@ public class DataSourceIntegrityIngestModule implements DataSourceIngestModule {
* Utility class to hold data for a specific hash algorithm.
*/
private class HashData {
private HashType type;
private MessageDigest digest;
private String storedHash;

View File

@ -64,6 +64,7 @@ import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Account;
import org.sleuthkit.datamodel.Blackboard.BlackboardException;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardArtifactTag;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.CaseDbAccessManager;
@ -74,7 +75,9 @@ import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.Host;
import org.sleuthkit.datamodel.Image;
import org.sleuthkit.datamodel.LocalFilesDataSource;
import org.sleuthkit.datamodel.OsAccount;
import org.sleuthkit.datamodel.Pool;
import org.sleuthkit.datamodel.Score;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction;
import org.sleuthkit.datamodel.TagName;
@ -870,8 +873,9 @@ public class PortableCaseReportModule implements ReportModule {
/**
* Add all artifacts with a given tag to the portable case.
*
* @param oldTagName The TagName object from the current case
* @param progressPanel The progress panel
* @param dataSourceId The data source id.
* @param oldTagName The TagName object from the current case.
* @param progressPanel The progress panel.
*
* @throws TskCoreException
*/
@ -892,7 +896,10 @@ public class PortableCaseReportModule implements ReportModule {
long newContentId = copyContentToPortableCase(content, progressPanel);
// Copy the artifact
BlackboardArtifact newArtifact = copyArtifact(newContentId, tag.getArtifact());
Long dataSourceObjId = content == null || content.getDataSource() == null
? null
: content.getDataSource().getId();
BlackboardArtifact newArtifact = copyArtifact(dataSourceObjId, newContentId, tag.getArtifact());
// Copy any attachments
copyAttachments(newArtifact, tag.getArtifact(), portableSkCase.getAbstractFileById(newContentId));
@ -912,15 +919,16 @@ public class PortableCaseReportModule implements ReportModule {
* Copy an artifact into the new case. Will also copy any associated
* artifacts
*
* @param newContentId The content ID (in the portable case) of the source
* content
* @param artifactToCopy The artifact to copy
* @param newDataSourceId The data source ID (in the portable case).
* @param newContentId The content ID (in the portable case) of the
* source content.
* @param artifactToCopy The artifact to copy.
*
* @return The new artifact in the portable case
* @return The new artifact in the portable case.
*
* @throws TskCoreException
*/
private BlackboardArtifact copyArtifact(long newContentId, BlackboardArtifact artifactToCopy) throws TskCoreException {
private BlackboardArtifact copyArtifact(Long newDataSourceId, long newContentId, BlackboardArtifact artifactToCopy) throws TskCoreException {
if (oldArtifactIdToNewArtifact.containsKey(artifactToCopy.getArtifactID())) {
return oldArtifactIdToNewArtifact.get(artifactToCopy.getArtifactID());
@ -931,14 +939,11 @@ public class PortableCaseReportModule implements ReportModule {
List<BlackboardAttribute> newAttrs = new ArrayList<>();
if (oldAssociatedAttribute != null) {
BlackboardArtifact oldAssociatedArtifact = currentCase.getSleuthkitCase().getBlackboardArtifact(oldAssociatedAttribute.getValueLong());
BlackboardArtifact newAssociatedArtifact = copyArtifact(newContentId, oldAssociatedArtifact);
BlackboardArtifact newAssociatedArtifact = copyArtifact(newDataSourceId, newContentId, oldAssociatedArtifact);
newAttrs.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT,
String.join(",", oldAssociatedAttribute.getSources()), newAssociatedArtifact.getArtifactID()));
}
// Create the new artifact
int newArtifactTypeId = getNewArtifactTypeId(artifactToCopy);
BlackboardArtifact newArtifact = portableSkCase.newBlackboardArtifact(newArtifactTypeId, newContentId);
List<BlackboardAttribute> oldAttrs = artifactToCopy.getAttributes();
// Copy over each attribute, making sure the type is in the new case.
@ -978,7 +983,20 @@ public class PortableCaseReportModule implements ReportModule {
}
}
newArtifact.addAttributes(newAttrs);
// Create the new artifact
int newArtifactTypeId = getNewArtifactTypeId(artifactToCopy);
BlackboardArtifact.Type type = new BlackboardArtifact.Type(ARTIFACT_TYPE.fromID(newArtifactTypeId));
BlackboardArtifact newArtifact = null;
switch (type.getCategory()) {
case ANALYSIS_RESULT:
newArtifact = portableSkCase.getBlackboard().newDataArtifact(type, newContentId, newDataSourceId, newAttrs, osAccount);
break;
case DATA_ARTIFACT:
newArtifact = portableSkCase.getBlackboard().newAnalysisResult(type, newContentId, newDataSourceId, Score.SCORE_UNKNOWN, null, null, null, newAttrs);
break;
default:
throw new TskCoreException("Unknown category: " + type.getCategory());
}
oldArtifactIdToNewArtifact.put(artifactToCopy.getArtifactID(), newArtifact);
return newArtifact;
@ -1075,10 +1093,12 @@ public class PortableCaseReportModule implements ReportModule {
parentId = copyContent(content.getParent());
}
Long dataSourceObjId = content.getDataSource() == null ? null : content.getDataSource().getId();
Content newContent;
if (content instanceof BlackboardArtifact) {
BlackboardArtifact artifactToCopy = (BlackboardArtifact) content;
newContent = copyArtifact(parentId, artifactToCopy);
newContent = copyArtifact(dataSourceObjId, parentId, artifactToCopy);
} else {
// Get or create the host (if needed) before beginning transaction.
@ -1111,7 +1131,7 @@ public class PortableCaseReportModule implements ReportModule {
fs.getName(), trans);
} else if (content instanceof BlackboardArtifact) {
BlackboardArtifact artifactToCopy = (BlackboardArtifact) content;
newContent = copyArtifact(parentId, artifactToCopy);
newContent = copyArtifact(dataSourceObjId, parentId, artifactToCopy);
} else if (content instanceof AbstractFile) {
AbstractFile abstractFile = (AbstractFile) content;
@ -1170,7 +1190,8 @@ public class PortableCaseReportModule implements ReportModule {
/**
* Copy path ID attribute to new case along with the referenced file.
*
* @param newArtifact The new artifact in the portable case. Should not have a TSK_PATH_ID attribute.
* @param newArtifact The new artifact in the portable case. Should not have
* a TSK_PATH_ID attribute.
* @param oldArtifact The old artifact.
*
* @throws TskCoreException
@ -1193,9 +1214,11 @@ public class PortableCaseReportModule implements ReportModule {
/**
* Copy attachments to the portable case.
*
* @param newArtifact The new artifact in the portable case. Should not have a TSK_ATTACHMENTS attribute.
* @param newArtifact The new artifact in the portable case. Should not have
* a TSK_ATTACHMENTS attribute.
* @param oldArtifact The old artifact.
* @param newFile The new file in the portable case associated with the artifact.
* @param newFile The new file in the portable case associated with the
* artifact.
*
* @throws TskCoreException
*/
@ -1232,8 +1255,7 @@ public class PortableCaseReportModule implements ReportModule {
CommunicationArtifactsHelper communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(),
newSourceStr, newFile, Account.Type.EMAIL);
communicationArtifactsHelper.addAttachments(newArtifact, new MessageAttachments(newFileAttachments, msgAttachments.getUrlAttachments()));
}
catch (BlackboardJsonAttrUtil.InvalidJsonException ex) {
} catch (BlackboardJsonAttrUtil.InvalidJsonException ex) {
throw new TskCoreException(String.format("Unable to parse json for MessageAttachments object in artifact: %s", oldArtifact.getName()), ex);
}
} else { // backward compatibility - email message attachments are derived files, children of the message.