diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java index 13877d701d..c982220e8b 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java @@ -62,6 +62,7 @@ import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.HashUtility; import org.sleuthkit.datamodel.HashUtility.HashResult; import org.sleuthkit.datamodel.HashUtility.HashType; +import org.sleuthkit.datamodel.IngestJobInfo; import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -137,7 +138,6 @@ public class MalwareScanIngestModule implements FileIngestModule { // TODO minimize state // private RunState runState = null; // private IngestJobState ingestJobState = null; - @Messages({ "MalwareScanIngestModule_malwareTypeDisplayName=Malware", "MalwareScanIngestModule_ShareProcessing_noLicense_title=No Cyber Triage License", @@ -167,81 +167,84 @@ public class MalwareScanIngestModule implements FileIngestModule { throw new IngestModuleException("An exception occurred on MalwareScanIngestModule startup", ex); } } - + /** * Sets up the state necessary for a new ingest job. + * * @param context The ingest job context. - * @return A pair of the runtime state (i.e. started up, disabled) and parameters required for the job. - * @throws Exception + * @return A pair of the runtime state (i.e. started up, disabled) and + * parameters required for the job. + * @throws Exception */ private Pair getNewJobState(IngestJobContext context) throws Exception { - // get saved license - Optional licenseInfoOpt = ctSettingsPersistence.loadLicenseInfo(); - if (licenseInfoOpt.isEmpty() || licenseInfoOpt.get().getDecryptedLicense() == null) { - notifyWarning( - Bundle.MalwareScanIngestModule_ShareProcessing_noLicense_title(), - Bundle.MalwareScanIngestModule_ShareProcessing_noLicense_desc(), - null); - return Pair.of(RunState.DISABLED, null); - } + // get saved license + Optional licenseInfoOpt = ctSettingsPersistence.loadLicenseInfo(); + if (licenseInfoOpt.isEmpty() || licenseInfoOpt.get().getDecryptedLicense() == null) { + notifyWarning( + Bundle.MalwareScanIngestModule_ShareProcessing_noLicense_title(), + Bundle.MalwareScanIngestModule_ShareProcessing_noLicense_desc(), + null); + return Pair.of(RunState.DISABLED, null); + } - AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfoOpt.get().getDecryptedLicense()); - // syncronously fetch malware scans info + AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfoOpt.get().getDecryptedLicense()); + // syncronously fetch malware scans info - // determine lookups remaining - long lookupsRemaining = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount()); - if (lookupsRemaining <= 0) { + // determine lookups remaining + long lookupsRemaining = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount()); + if (lookupsRemaining <= 0) { + notifyWarning( + Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_title(), + Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_desc(), + null); + return Pair.of(RunState.DISABLED, null); + } else if (lookupsRemaining < LOW_LOOKUPS_REMAINING) { + notifyWarning( + Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_title(), + Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_desc(lookupsRemaining), + null); + } + + // determine lookups remaining + boolean uploadFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles(); + if (uploadFiles) { + long uploadsRemaining = remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()); + if (uploadsRemaining <= 0) { notifyWarning( - Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_title(), - Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_desc(), + Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_title(), + Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_desc(), null); - return Pair.of(RunState.DISABLED, null); - } else if (lookupsRemaining < LOW_LOOKUPS_REMAINING) { + uploadFiles = false; + } else if (lookupsRemaining < LOW_UPLOADS_REMAINING) { notifyWarning( - Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_title(), - Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_desc(lookupsRemaining), + Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_title(), + Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_desc(lookupsRemaining), null); } + } - // determine lookups remaining - boolean uploadFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles(); - if (uploadFiles) { - long uploadsRemaining = remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()); - if (uploadsRemaining <= 0) { - notifyWarning( - Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_title(), - Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_desc(), - null); - uploadFiles = false; - } else if (lookupsRemaining < LOW_UPLOADS_REMAINING) { - notifyWarning( - Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_title(), - Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_desc(lookupsRemaining), - null); - } - } + // setup necessary variables for processing + SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase(); + BlackboardArtifact.Type malwareType = tskCase.getBlackboard().getOrAddArtifactType( + MALWARE_TYPE_NAME, + Bundle.MalwareScanIngestModule_malwareTypeDisplayName(), + BlackboardArtifact.Category.ANALYSIS_RESULT); - // setup necessary variables for processing - SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase(); - BlackboardArtifact.Type malwareType = tskCase.getBlackboard().getOrAddArtifactType( - MALWARE_TYPE_NAME, - Bundle.MalwareScanIngestModule_malwareTypeDisplayName(), - BlackboardArtifact.Category.ANALYSIS_RESULT); - - IngestJobState ingestJobState = new IngestJobState( - tskCase, - new FileTypeDetector(), - licenseInfoOpt.get(), - malwareType, - context.getDataSource().getId(), - context.getJobId(), - uploadFiles); - - return Pair.of(RunState.STARTED_UP, ingestJobState); + IngestJobState ingestJobState = new IngestJobState( + tskCase, + new FileTypeDetector(), + licenseInfoOpt.get(), + malwareType, + context.getDataSource().getId(), + context.getJobId(), + uploadFiles); + + return Pair.of(RunState.STARTED_UP, ingestJobState); } /** * Determines remaining given a possibly null limit and used count. + * * @param limit The limit (can be null). * @param used The number used (can be null). * @return The remaining amount. @@ -254,6 +257,7 @@ public class MalwareScanIngestModule implements FileIngestModule { /** * Gets the md5 hash from the abstract file or calculates it. + * * @param af The abstract file. * @return The md5 hash (or null if could not be determined). */ @@ -282,6 +286,7 @@ public class MalwareScanIngestModule implements FileIngestModule { return null; } + // TODO return to this @Messages({ "MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout", "MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out" @@ -314,6 +319,7 @@ public class MalwareScanIngestModule implements FileIngestModule { } } + // TODO return to this @Messages({ "MalwareScanIngestModule_SharedProcessing_authTokenResponseError_title=Authentication API error", "# {0} - errorResponse", @@ -338,7 +344,7 @@ public class MalwareScanIngestModule implements FileIngestModule { continue; } - String sanitizedMd5 = sanitizedMd5(fr.getMd5hash()); + String sanitizedMd5 = normalizedMd5(fr.getMd5hash()); md5ToObjId .computeIfAbsent(sanitizedMd5, (k) -> new ArrayList<>()) .add(fr.getObjId()); @@ -363,7 +369,7 @@ public class MalwareScanIngestModule implements FileIngestModule { // if being scanned, check list to run later List beingScannedList = statusGroupings.get(Status.BEING_SCANNED); processMissing(beingScannedList, md5ToObjId, false); - + // if not found, try upload List notFound = statusGroupings.get(Status.NOT_FOUND); processMissing(notFound, md5ToObjId, true); @@ -390,10 +396,11 @@ public class MalwareScanIngestModule implements FileIngestModule { } } - private static void processMissing(Collection results, Map> md5ToObjId, boolean doFileUpload) throws CTCloudException, TskCoreException { + // TODO return to this + private void processMissing(Collection results, Map> md5ToObjId, boolean doFileUpload) throws CTCloudException, TskCoreException { for (CTCloudBean beingScanned : CollectionUtils.emptyIfNull(results)) { - String sanitizedMd5 = sanitizedMd5(beingScanned.getMd5HashValue()); + String sanitizedMd5 = normalizedMd5(beingScanned.getMd5HashValue()); if (StringUtils.isBlank(sanitizedMd5)) { continue; } @@ -409,43 +416,7 @@ public class MalwareScanIngestModule implements FileIngestModule { } } - private void createArtifacts(List repResult, Map> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException { - List createdArtifacts = new ArrayList<>(); - if (!CollectionUtils.isEmpty(repResult)) { - SleuthkitCase.CaseDbTransaction trans = null; - try { - trans = tskCase.beginTransaction(); - for (CTCloudBean result : repResult) { - String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue()); - List objIds = md5ToObjId.remove(sanitizedMd5); - if (objIds == null || objIds.isEmpty()) { - continue; - } - - for (Long objId : objIds) { - AnalysisResult res = createAnalysisResult(objId, result, trans); - if (res != null) { - createdArtifacts.add(res); - } - } - } - - trans.commit(); - trans = null; - } finally { - if (trans != null) { - trans.rollback(); - createdArtifacts.clear(); - trans = null; - } - } - - if (!CollectionUtils.isEmpty(createdArtifacts)) { - tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId); - } - } - } - + // TODO return to this private List getHashLookupResults(List md5Hashes) throws CTCloudException { // get an auth token with the license AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense()); @@ -468,23 +439,43 @@ public class MalwareScanIngestModule implements FileIngestModule { ); } - private String sanitizedMd5(String orig) { + /** + * Normalizes an md5 string for the purposes of lookup in a map. + * @param orig The original value. + * @return The normalized value + */ + private static String normalizedMd5(String orig) { return StringUtils.defaultString(orig).trim().toLowerCase(); } - private boolean isUnknown(CTCloudBean cloudBean) { + /** + * Returns true if the cloud bean indicates that the file is not + * currently known by CT cloud. + * + * @param cloudBean The cloud bean. + * @return True if not known by CT cloud. + */ + private static boolean isUnknown(CTCloudBean cloudBean) { return cloudBean != null && cloudBean.getMalwareResult() != null && cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND; } - private boolean isUploadable(AbstractFile af) { + /** + * Whether or not an abstract file meets the requirements to be + * uploaded. + * + * @param af The abstract file. + * @return True if can be uploaded. + */ + private static boolean isUploadable(AbstractFile af) { long size = af.getSize(); return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE; } - private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException { - if (!uploadUnknownFiles) { + // TODO return to this + private boolean uploadFile(IngestJobState ingestJobState, CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException { + if (!ingestJobState.uploadUnknownFiles()) { return false; } @@ -492,7 +483,7 @@ public class MalwareScanIngestModule implements FileIngestModule { return false; } - AbstractFile af = tskCase.getAbstractFileById(objId); + AbstractFile af = ingestJobState.getTskCase().getAbstractFileById(objId); if (af == null) { return false; } @@ -502,12 +493,12 @@ public class MalwareScanIngestModule implements FileIngestModule { } // get auth token / file upload url - AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense(), true); + AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), true); if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) { throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR); } else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) { // don't proceed with upload if reached limit - uploadUnknownFiles = false; + ingestJobState.setUploadUnknownFiles(false); return false; } @@ -525,10 +516,11 @@ public class MalwareScanIngestModule implements FileIngestModule { .setSha1(af.getSha1Hash()) .setSha256(af.getSha256Hash()); - ctApiDAO.uploadMeta(new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), metaRequest); + ctApiDAO.uploadMeta(new AuthenticatedRequestData(ingestJobState.getLicenseInfo().getDecryptedLicense(), authTokenResponse), metaRequest); return true; } + // TODO return to this private boolean getUploadedFileResults(Map> md5objIdMapping) throws InterruptedException, CTCloudException, Blackboard.BlackboardException, TskCoreException { Map> remaining = new HashMap<>(md5objIdMapping); @@ -549,12 +541,83 @@ public class MalwareScanIngestModule implements FileIngestModule { return false; } + /** + * Creates TSK_MALWARE analysis results based on a list of cloud beans + * received from the CT cloud api. + * + * @param ingestJobState The ingest job state. + * @param repResult The list of cloud beans. Only cloud beans with a + * malware status + * @param md5ToObjId + * @throws org.sleuthkit.datamodel.Blackboard.BlackboardException + * @throws TskCoreException + */ + private void createAnalysisResults(IngestJobState ingestJobState, List repResult, Map> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException { + List createdArtifacts = new ArrayList<>(); + if (!CollectionUtils.isEmpty(repResult)) { + SleuthkitCase.CaseDbTransaction trans = null; + try { + trans = ingestJobState.getTskCase().beginTransaction(); + for (CTCloudBean result : repResult) { + String sanitizedMd5 = normalizedMd5(result.getMd5HashValue()); + List objIds = md5ToObjId.remove(sanitizedMd5); + if (objIds == null || objIds.isEmpty()) { + continue; + } + + for (Long objId : objIds) { + AnalysisResult res = createAnalysisResult(ingestJobState, trans, result, objId); + if (res != null) { + createdArtifacts.add(res); + } + } + } + + trans.commit(); + trans = null; + } finally { + if (trans != null) { + trans.rollback(); + createdArtifacts.clear(); + trans = null; + } + } + + if (!CollectionUtils.isEmpty(createdArtifacts)) { + ingestJobState.getTskCase().getBlackboard().postArtifacts( + createdArtifacts, + Bundle.MalwareScanIngestModuleFactory_displayName(), + ingestJobState.getIngestJobId() + ); + } + } + } + + /** + * Creates an analysis result for the given information. + * + * @param ingestJobState The state of the ingest job. + * @param trans The case database transaction to use. + * @param cloudBean The bean indicating the malware result. + * @param objId The object id of the corresponding file that will + * receive the analysis result. + * @return The created analysis result or null if none created. + * @throws org.sleuthkit.datamodel.Blackboard.BlackboardException + */ @Messages({ "MalwareScanIngestModule_SharedProcessing_createAnalysisResult_Yes=YES", "MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No=NO" }) - private AnalysisResult createAnalysisResult(Long objId, CTCloudBean cloudBean, SleuthkitCase.CaseDbTransaction trans) throws Blackboard.BlackboardException { - if (objId == null || cloudBean == null || cloudBean.getMalwareResult() == null) { + private AnalysisResult createAnalysisResult(IngestJobState ingestJobState, SleuthkitCase.CaseDbTransaction trans, CTCloudBean cloudBean, Long objId) throws Blackboard.BlackboardException { + if (objId == null || cloudBean == null || cloudBean.getMalwareResult() == null || cloudBean.getMalwareResult().getStatus() != Status.FOUND) { + logger.log(Level.WARNING, MessageFormat.format("Attempting to create analysis result with invalid parameters [objId: {0}, cloud bean status: {1}]", + objId == null + ? "" + : objId, + (cloudBean == null || cloudBean.getMalwareResult() == null || cloudBean.getMalwareResult().getStatus() == null) + ? "" + : cloudBean.getMalwareResult().getStatus().name() + )); return null; } @@ -568,10 +631,10 @@ public class MalwareScanIngestModule implements FileIngestModule { String justification = cloudBean.getMalwareResult().getStatusDescription(); - return tskCase.getBlackboard().newAnalysisResult( - malwareType, + return ingestJobState.getTskCase().getBlackboard().newAnalysisResult( + ingestJobState.getMalwareType(), objId, - dsId, + ingestJobState.getDsId(), score, conclusion, MALWARE_CONFIG, @@ -580,6 +643,9 @@ public class MalwareScanIngestModule implements FileIngestModule { trans).getAnalysisResult(); } + /** + * Called when ingest should shut down. + */ @Messages({ "MalwareScanIngestModule_SharedProcessing_flushTimeout_title=Processing Timeout", "MalwareScanIngestModule_SharedProcessing_flushTimeout_desc=A timeout occurred while finishing processing" @@ -593,7 +659,7 @@ public class MalwareScanIngestModule implements FileIngestModule { // flush any remaining items try { batchProcessor.flushAndReset(); - + getUploadedFileResults(this.unidentifiedHashes); } catch (InterruptedException ex) { notifyWarning( @@ -611,7 +677,15 @@ public class MalwareScanIngestModule implements FileIngestModule { } } - private void notifyWarning(String title, String message, Exception ex) { + /** + * Creates a warning notification to display in the lower right corner + * and a corresponding log message. + * + * @param title The title of the warning. + * @param message The message of the warning. + * @param ex The corresponding exception (or null if none). + */ + private static void notifyWarning(String title, String message, Exception ex) { MessageNotifyUtil.Notify.warn(title, message); logger.log(Level.WARNING, message, ex); } @@ -639,8 +713,9 @@ public class MalwareScanIngestModule implements FileIngestModule { } } - + class IngestJobState { + private final SleuthkitCase tskCase; private final FileTypeDetector fileTypeDetector; private final LicenseInfo licenseInfo; @@ -648,7 +723,7 @@ public class MalwareScanIngestModule implements FileIngestModule { private final long dsId; private final long ingestJobId; private final Map> unidentifiedHashes = new HashMap<>(); - + // this can change mid run private boolean uploadUnknownFiles;