diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java index e249b487d5..2a67b46792 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java @@ -26,6 +26,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean; import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo; import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean; +import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status; import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest; import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence; import java.text.MessageFormat; @@ -103,7 +104,6 @@ public class MalwareScanIngestModule implements FileIngestModule { private static final long MAX_UPLOAD_SIZE = 1_000_000_000; private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5; private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000; - private static final Set EXECUTABLE_MIME_TYPES = Stream.of( "application/x-bat",//NON-NLS @@ -128,6 +128,7 @@ public class MalwareScanIngestModule implements FileIngestModule { private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance(); private final CTApiDAO ctApiDAO = CTApiDAO.getInstance(); + // TODO minimize state private RunState runState = null; private SleuthkitCase tskCase = null; @@ -137,6 +138,7 @@ public class MalwareScanIngestModule implements FileIngestModule { private long dsId = 0; private long ingestJobId = 0; private boolean uploadUnknownFiles = false; + private Map> unidentifiedHashes = null; @Messages({ "MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low", @@ -196,7 +198,8 @@ public class MalwareScanIngestModule implements FileIngestModule { ingestJobId = context.getJobId(); licenseInfo = licenseInfoOpt.get(); uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles(); - + unidentifiedHashes = new HashMap<>(); + // set run state to initialized runState = RunState.STARTED_UP; } catch (Exception ex) { @@ -306,60 +309,14 @@ public class MalwareScanIngestModule implements FileIngestModule { } try { - // get an auth token with the license - AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense()); - - // make sure we are in bounds for the remaining scans - long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount()); - if (remainingScans <= 0) { - runState = RunState.DISABLED; - notifyWarning( - Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(), - Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(), - null); - return; - } - - // using auth token, get results - List repResult = ctApiDAO.getReputationResults( - new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), - md5Hashes - ); - - List createdArtifacts = new ArrayList<>(); - if (!CollectionUtils.isEmpty(repResult)) { - SleuthkitCase.CaseDbTransaction trans = null; - try { - trans = tskCase.beginTransaction(); - for (CTCloudBean result : repResult) { - String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue()); - List objIds = md5ToObjId.remove(sanitizedMd5); - if (objIds == null || objIds.isEmpty()) { - continue; - } - - for (Long objId : objIds) { - AnalysisResult res = createAnalysisResult(objId, result, trans); - if (res != null) { - createdArtifacts.add(res); - } - } - } - - trans.commit(); - trans = null; - } finally { - if (trans != null) { - trans.rollback(); - createdArtifacts.clear(); - trans = null; - } - } - - if (!CollectionUtils.isEmpty(createdArtifacts)) { - tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId); - } - } + List repResult = getHashLookupResults(md5Hashes); + Map> partitioned = repResult.stream() + .filter(bean -> bean.getMalwareResult() != null) + .collect(Collectors.partitioningBy(bean -> bean.getMalwareResult().getStatus() == Status.FOUND)); + + // TODO handle caching list and creating new items + + createArtifacts(repResult, md5ToObjId); } catch (Exception ex) { notifyWarning( Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(), @@ -368,6 +325,65 @@ public class MalwareScanIngestModule implements FileIngestModule { } } + private void createArtifacts(List repResult, Map> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException { + List createdArtifacts = new ArrayList<>(); + if (!CollectionUtils.isEmpty(repResult)) { + SleuthkitCase.CaseDbTransaction trans = null; + try { + trans = tskCase.beginTransaction(); + for (CTCloudBean result : repResult) { + String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue()); + List objIds = md5ToObjId.remove(sanitizedMd5); + if (objIds == null || objIds.isEmpty()) { + continue; + } + + for (Long objId : objIds) { + AnalysisResult res = createAnalysisResult(objId, result, trans); + if (res != null) { + createdArtifacts.add(res); + } + } + } + + trans.commit(); + trans = null; + } finally { + if (trans != null) { + trans.rollback(); + createdArtifacts.clear(); + trans = null; + } + } + + if (!CollectionUtils.isEmpty(createdArtifacts)) { + tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId); + } + } + } + + private List getHashLookupResults(List md5Hashes) throws CTCloudException { + // get an auth token with the license + AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense()); + + // make sure we are in bounds for the remaining scans + long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount()); + if (remainingScans <= 0) { + runState = RunState.DISABLED; + notifyWarning( + Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(), + Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(), + null); + return Collections.emptyList(); + } + + // using auth token, get results + return ctApiDAO.getReputationResults( + new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), + md5Hashes + ); + } + private String sanitizedMd5(String orig) { return StringUtils.defaultString(orig).trim().toLowerCase(); } @@ -392,7 +408,7 @@ public class MalwareScanIngestModule implements FileIngestModule { return false; } - AbstractFile af = skCase.getAbstractFileById(objId); + AbstractFile af = tskCase.getAbstractFileById(objId); if (af == null) { return false; } @@ -402,7 +418,7 @@ public class MalwareScanIngestModule implements FileIngestModule { } // get auth token / file upload url - AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true); + AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense(), true); if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) { throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR); } else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) { @@ -425,27 +441,29 @@ public class MalwareScanIngestModule implements FileIngestModule { .setSha1(af.getSha1Hash()) .setSha256(af.getSha256Hash()); - ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest); + ctApiDAO.uploadMeta(new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), metaRequest); return true; } - private boolean getUploadedFileResults(Map> md5objIdMapping) { + private boolean getUploadedFileResults(Map> md5objIdMapping) throws InterruptedException, CTCloudException, Blackboard.BlackboardException, TskCoreException { + // TODO integrate this Map> remaining = new HashMap<>(md5objIdMapping); for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) { List> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE); for (List batch : md5Batches) { - // TODO query and capture still unknown + List repResult = getHashLookupResults(batch); + createArtifacts(repResult, remaining); } - + if (remaining.isEmpty()) { return true; } - - + Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS); } + return false; } @Messages({