refactoring and commenting

This commit is contained in:
Greg DiCristofaro 2023-07-26 20:42:37 -04:00
parent 10ae3411ac
commit 19385ef3f8

View File

@ -62,6 +62,7 @@ import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.HashUtility.HashResult;
import org.sleuthkit.datamodel.HashUtility.HashType;
import org.sleuthkit.datamodel.IngestJobInfo;
import org.sleuthkit.datamodel.Score;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
@ -137,7 +138,6 @@ public class MalwareScanIngestModule implements FileIngestModule {
// TODO minimize state
// private RunState runState = null;
// private IngestJobState ingestJobState = null;
@Messages({
"MalwareScanIngestModule_malwareTypeDisplayName=Malware",
"MalwareScanIngestModule_ShareProcessing_noLicense_title=No Cyber Triage License",
@ -167,81 +167,84 @@ public class MalwareScanIngestModule implements FileIngestModule {
throw new IngestModuleException("An exception occurred on MalwareScanIngestModule startup", ex);
}
}
/**
* Sets up the state necessary for a new ingest job.
*
* @param context The ingest job context.
* @return A pair of the runtime state (i.e. started up, disabled) and parameters required for the job.
* @throws Exception
* @return A pair of the runtime state (i.e. started up, disabled) and
* parameters required for the job.
* @throws Exception
*/
private Pair<RunState, IngestJobState> getNewJobState(IngestJobContext context) throws Exception {
// get saved license
Optional<LicenseInfo> licenseInfoOpt = ctSettingsPersistence.loadLicenseInfo();
if (licenseInfoOpt.isEmpty() || licenseInfoOpt.get().getDecryptedLicense() == null) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_noLicense_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_noLicense_desc(),
null);
return Pair.of(RunState.DISABLED, null);
}
// get saved license
Optional<LicenseInfo> licenseInfoOpt = ctSettingsPersistence.loadLicenseInfo();
if (licenseInfoOpt.isEmpty() || licenseInfoOpt.get().getDecryptedLicense() == null) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_noLicense_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_noLicense_desc(),
null);
return Pair.of(RunState.DISABLED, null);
}
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfoOpt.get().getDecryptedLicense());
// syncronously fetch malware scans info
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfoOpt.get().getDecryptedLicense());
// syncronously fetch malware scans info
// determine lookups remaining
long lookupsRemaining = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
if (lookupsRemaining <= 0) {
// determine lookups remaining
long lookupsRemaining = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
if (lookupsRemaining <= 0) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_desc(),
null);
return Pair.of(RunState.DISABLED, null);
} else if (lookupsRemaining < LOW_LOOKUPS_REMAINING) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_desc(lookupsRemaining),
null);
}
// determine lookups remaining
boolean uploadFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
if (uploadFiles) {
long uploadsRemaining = remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount());
if (uploadsRemaining <= 0) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_desc(),
Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_desc(),
null);
return Pair.of(RunState.DISABLED, null);
} else if (lookupsRemaining < LOW_LOOKUPS_REMAINING) {
uploadFiles = false;
} else if (lookupsRemaining < LOW_UPLOADS_REMAINING) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_desc(lookupsRemaining),
Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_desc(lookupsRemaining),
null);
}
}
// determine lookups remaining
boolean uploadFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
if (uploadFiles) {
long uploadsRemaining = remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount());
if (uploadsRemaining <= 0) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_desc(),
null);
uploadFiles = false;
} else if (lookupsRemaining < LOW_UPLOADS_REMAINING) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_desc(lookupsRemaining),
null);
}
}
// setup necessary variables for processing
SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase();
BlackboardArtifact.Type malwareType = tskCase.getBlackboard().getOrAddArtifactType(
MALWARE_TYPE_NAME,
Bundle.MalwareScanIngestModule_malwareTypeDisplayName(),
BlackboardArtifact.Category.ANALYSIS_RESULT);
// setup necessary variables for processing
SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase();
BlackboardArtifact.Type malwareType = tskCase.getBlackboard().getOrAddArtifactType(
MALWARE_TYPE_NAME,
Bundle.MalwareScanIngestModule_malwareTypeDisplayName(),
BlackboardArtifact.Category.ANALYSIS_RESULT);
IngestJobState ingestJobState = new IngestJobState(
tskCase,
new FileTypeDetector(),
licenseInfoOpt.get(),
malwareType,
context.getDataSource().getId(),
context.getJobId(),
uploadFiles);
return Pair.of(RunState.STARTED_UP, ingestJobState);
IngestJobState ingestJobState = new IngestJobState(
tskCase,
new FileTypeDetector(),
licenseInfoOpt.get(),
malwareType,
context.getDataSource().getId(),
context.getJobId(),
uploadFiles);
return Pair.of(RunState.STARTED_UP, ingestJobState);
}
/**
* Determines remaining given a possibly null limit and used count.
*
* @param limit The limit (can be null).
* @param used The number used (can be null).
* @return The remaining amount.
@ -254,6 +257,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
/**
* Gets the md5 hash from the abstract file or calculates it.
*
* @param af The abstract file.
* @return The md5 hash (or null if could not be determined).
*/
@ -282,6 +286,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
return null;
}
// TODO return to this
@Messages({
"MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout",
"MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out"
@ -314,6 +319,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
}
}
// TODO return to this
@Messages({
"MalwareScanIngestModule_SharedProcessing_authTokenResponseError_title=Authentication API error",
"# {0} - errorResponse",
@ -338,7 +344,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
continue;
}
String sanitizedMd5 = sanitizedMd5(fr.getMd5hash());
String sanitizedMd5 = normalizedMd5(fr.getMd5hash());
md5ToObjId
.computeIfAbsent(sanitizedMd5, (k) -> new ArrayList<>())
.add(fr.getObjId());
@ -363,7 +369,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
// if being scanned, check list to run later
List<CTCloudBean> beingScannedList = statusGroupings.get(Status.BEING_SCANNED);
processMissing(beingScannedList, md5ToObjId, false);
// if not found, try upload
List<CTCloudBean> notFound = statusGroupings.get(Status.NOT_FOUND);
processMissing(notFound, md5ToObjId, true);
@ -390,10 +396,11 @@ public class MalwareScanIngestModule implements FileIngestModule {
}
}
private static void processMissing(Collection<CTCloudBean> results, Map<String, List<Long>> md5ToObjId, boolean doFileUpload) throws CTCloudException, TskCoreException {
// TODO return to this
private void processMissing(Collection<CTCloudBean> results, Map<String, List<Long>> md5ToObjId, boolean doFileUpload) throws CTCloudException, TskCoreException {
for (CTCloudBean beingScanned : CollectionUtils.emptyIfNull(results)) {
String sanitizedMd5 = sanitizedMd5(beingScanned.getMd5HashValue());
String sanitizedMd5 = normalizedMd5(beingScanned.getMd5HashValue());
if (StringUtils.isBlank(sanitizedMd5)) {
continue;
}
@ -409,43 +416,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
}
}
private void createArtifacts(List<CTCloudBean> repResult, Map<String, List<Long>> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException {
List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
if (!CollectionUtils.isEmpty(repResult)) {
SleuthkitCase.CaseDbTransaction trans = null;
try {
trans = tskCase.beginTransaction();
for (CTCloudBean result : repResult) {
String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue());
List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
if (objIds == null || objIds.isEmpty()) {
continue;
}
for (Long objId : objIds) {
AnalysisResult res = createAnalysisResult(objId, result, trans);
if (res != null) {
createdArtifacts.add(res);
}
}
}
trans.commit();
trans = null;
} finally {
if (trans != null) {
trans.rollback();
createdArtifacts.clear();
trans = null;
}
}
if (!CollectionUtils.isEmpty(createdArtifacts)) {
tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
}
}
}
// TODO return to this
private List<CTCloudBean> getHashLookupResults(List<String> md5Hashes) throws CTCloudException {
// get an auth token with the license
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense());
@ -468,23 +439,43 @@ public class MalwareScanIngestModule implements FileIngestModule {
);
}
private String sanitizedMd5(String orig) {
/**
* Normalizes an md5 string for the purposes of lookup in a map.
* @param orig The original value.
* @return The normalized value
*/
private static String normalizedMd5(String orig) {
return StringUtils.defaultString(orig).trim().toLowerCase();
}
private boolean isUnknown(CTCloudBean cloudBean) {
/**
* Returns true if the cloud bean indicates that the file is not
* currently known by CT cloud.
*
* @param cloudBean The cloud bean.
* @return True if not known by CT cloud.
*/
private static boolean isUnknown(CTCloudBean cloudBean) {
return cloudBean != null
&& cloudBean.getMalwareResult() != null
&& cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND;
}
private boolean isUploadable(AbstractFile af) {
/**
* Whether or not an abstract file meets the requirements to be
* uploaded.
*
* @param af The abstract file.
* @return True if can be uploaded.
*/
private static boolean isUploadable(AbstractFile af) {
long size = af.getSize();
return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
}
private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
if (!uploadUnknownFiles) {
// TODO return to this
private boolean uploadFile(IngestJobState ingestJobState, CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
if (!ingestJobState.uploadUnknownFiles()) {
return false;
}
@ -492,7 +483,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
return false;
}
AbstractFile af = tskCase.getAbstractFileById(objId);
AbstractFile af = ingestJobState.getTskCase().getAbstractFileById(objId);
if (af == null) {
return false;
}
@ -502,12 +493,12 @@ public class MalwareScanIngestModule implements FileIngestModule {
}
// get auth token / file upload url
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense(), true);
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), true);
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
} else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
// don't proceed with upload if reached limit
uploadUnknownFiles = false;
ingestJobState.setUploadUnknownFiles(false);
return false;
}
@ -525,10 +516,11 @@ public class MalwareScanIngestModule implements FileIngestModule {
.setSha1(af.getSha1Hash())
.setSha256(af.getSha256Hash());
ctApiDAO.uploadMeta(new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), metaRequest);
ctApiDAO.uploadMeta(new AuthenticatedRequestData(ingestJobState.getLicenseInfo().getDecryptedLicense(), authTokenResponse), metaRequest);
return true;
}
// TODO return to this
private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) throws InterruptedException, CTCloudException, Blackboard.BlackboardException, TskCoreException {
Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping);
@ -549,12 +541,83 @@ public class MalwareScanIngestModule implements FileIngestModule {
return false;
}
/**
* Creates TSK_MALWARE analysis results based on a list of cloud beans
* received from the CT cloud api.
*
* @param ingestJobState The ingest job state.
* @param repResult The list of cloud beans. Only cloud beans with a
* malware status
* @param md5ToObjId
* @throws org.sleuthkit.datamodel.Blackboard.BlackboardException
* @throws TskCoreException
*/
private void createAnalysisResults(IngestJobState ingestJobState, List<CTCloudBean> repResult, Map<String, List<Long>> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException {
List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
if (!CollectionUtils.isEmpty(repResult)) {
SleuthkitCase.CaseDbTransaction trans = null;
try {
trans = ingestJobState.getTskCase().beginTransaction();
for (CTCloudBean result : repResult) {
String sanitizedMd5 = normalizedMd5(result.getMd5HashValue());
List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
if (objIds == null || objIds.isEmpty()) {
continue;
}
for (Long objId : objIds) {
AnalysisResult res = createAnalysisResult(ingestJobState, trans, result, objId);
if (res != null) {
createdArtifacts.add(res);
}
}
}
trans.commit();
trans = null;
} finally {
if (trans != null) {
trans.rollback();
createdArtifacts.clear();
trans = null;
}
}
if (!CollectionUtils.isEmpty(createdArtifacts)) {
ingestJobState.getTskCase().getBlackboard().postArtifacts(
createdArtifacts,
Bundle.MalwareScanIngestModuleFactory_displayName(),
ingestJobState.getIngestJobId()
);
}
}
}
/**
* Creates an analysis result for the given information.
*
* @param ingestJobState The state of the ingest job.
* @param trans The case database transaction to use.
* @param cloudBean The bean indicating the malware result.
* @param objId The object id of the corresponding file that will
* receive the analysis result.
* @return The created analysis result or null if none created.
* @throws org.sleuthkit.datamodel.Blackboard.BlackboardException
*/
@Messages({
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_Yes=YES",
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No=NO"
})
private AnalysisResult createAnalysisResult(Long objId, CTCloudBean cloudBean, SleuthkitCase.CaseDbTransaction trans) throws Blackboard.BlackboardException {
if (objId == null || cloudBean == null || cloudBean.getMalwareResult() == null) {
private AnalysisResult createAnalysisResult(IngestJobState ingestJobState, SleuthkitCase.CaseDbTransaction trans, CTCloudBean cloudBean, Long objId) throws Blackboard.BlackboardException {
if (objId == null || cloudBean == null || cloudBean.getMalwareResult() == null || cloudBean.getMalwareResult().getStatus() != Status.FOUND) {
logger.log(Level.WARNING, MessageFormat.format("Attempting to create analysis result with invalid parameters [objId: {0}, cloud bean status: {1}]",
objId == null
? "<null>"
: objId,
(cloudBean == null || cloudBean.getMalwareResult() == null || cloudBean.getMalwareResult().getStatus() == null)
? "<null>"
: cloudBean.getMalwareResult().getStatus().name()
));
return null;
}
@ -568,10 +631,10 @@ public class MalwareScanIngestModule implements FileIngestModule {
String justification = cloudBean.getMalwareResult().getStatusDescription();
return tskCase.getBlackboard().newAnalysisResult(
malwareType,
return ingestJobState.getTskCase().getBlackboard().newAnalysisResult(
ingestJobState.getMalwareType(),
objId,
dsId,
ingestJobState.getDsId(),
score,
conclusion,
MALWARE_CONFIG,
@ -580,6 +643,9 @@ public class MalwareScanIngestModule implements FileIngestModule {
trans).getAnalysisResult();
}
/**
* Called when ingest should shut down.
*/
@Messages({
"MalwareScanIngestModule_SharedProcessing_flushTimeout_title=Processing Timeout",
"MalwareScanIngestModule_SharedProcessing_flushTimeout_desc=A timeout occurred while finishing processing"
@ -593,7 +659,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
// flush any remaining items
try {
batchProcessor.flushAndReset();
getUploadedFileResults(this.unidentifiedHashes);
} catch (InterruptedException ex) {
notifyWarning(
@ -611,7 +677,15 @@ public class MalwareScanIngestModule implements FileIngestModule {
}
}
private void notifyWarning(String title, String message, Exception ex) {
/**
* Creates a warning notification to display in the lower right corner
* and a corresponding log message.
*
* @param title The title of the warning.
* @param message The message of the warning.
* @param ex The corresponding exception (or null if none).
*/
private static void notifyWarning(String title, String message, Exception ex) {
MessageNotifyUtil.Notify.warn(title, message);
logger.log(Level.WARNING, message, ex);
}
@ -639,8 +713,9 @@ public class MalwareScanIngestModule implements FileIngestModule {
}
}
class IngestJobState {
private final SleuthkitCase tskCase;
private final FileTypeDetector fileTypeDetector;
private final LicenseInfo licenseInfo;
@ -648,7 +723,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
private final long dsId;
private final long ingestJobId;
private final Map<String, List<Long>> unidentifiedHashes = new HashMap<>();
// this can change mid run
private boolean uploadUnknownFiles;