mirror of
https://github.com/overcuriousity/autopsy-flatpak.git
synced 2025-07-06 21:00:22 +00:00
updates
This commit is contained in:
Greg DiCristofaro
parent
f23331b64c
commit
032993c858
@ -26,6 +26,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean;
|
||||
import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse;
|
||||
import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
|
||||
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean;
|
||||
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
|
||||
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
|
||||
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
|
||||
import java.text.MessageFormat;
|
||||
@ -103,7 +104,6 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
|
||||
private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5;
|
||||
private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
|
||||
|
||||
|
||||
private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
|
||||
"application/x-bat",//NON-NLS
|
||||
@ -128,6 +128,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
|
||||
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
|
||||
|
||||
// TODO minimize state
|
||||
private RunState runState = null;
|
||||
|
||||
private SleuthkitCase tskCase = null;
|
||||
@ -137,6 +138,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
private long dsId = 0;
|
||||
private long ingestJobId = 0;
|
||||
private boolean uploadUnknownFiles = false;
|
||||
private Map<String, List<Long>> unidentifiedHashes = null;
|
||||
|
||||
@Messages({
|
||||
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
|
||||
@ -196,7 +198,8 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
ingestJobId = context.getJobId();
|
||||
licenseInfo = licenseInfoOpt.get();
|
||||
uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
|
||||
|
||||
unidentifiedHashes = new HashMap<>();
|
||||
|
||||
// set run state to initialized
|
||||
runState = RunState.STARTED_UP;
|
||||
} catch (Exception ex) {
|
||||
@ -306,60 +309,14 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
}
|
||||
|
||||
try {
|
||||
// get an auth token with the license
|
||||
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense());
|
||||
|
||||
// make sure we are in bounds for the remaining scans
|
||||
long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
|
||||
if (remainingScans <= 0) {
|
||||
runState = RunState.DISABLED;
|
||||
notifyWarning(
|
||||
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
|
||||
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
|
||||
null);
|
||||
return;
|
||||
}
|
||||
|
||||
// using auth token, get results
|
||||
List<CTCloudBean> repResult = ctApiDAO.getReputationResults(
|
||||
new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
|
||||
md5Hashes
|
||||
);
|
||||
|
||||
List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(repResult)) {
|
||||
SleuthkitCase.CaseDbTransaction trans = null;
|
||||
try {
|
||||
trans = tskCase.beginTransaction();
|
||||
for (CTCloudBean result : repResult) {
|
||||
String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue());
|
||||
List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
|
||||
if (objIds == null || objIds.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (Long objId : objIds) {
|
||||
AnalysisResult res = createAnalysisResult(objId, result, trans);
|
||||
if (res != null) {
|
||||
createdArtifacts.add(res);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trans.commit();
|
||||
trans = null;
|
||||
} finally {
|
||||
if (trans != null) {
|
||||
trans.rollback();
|
||||
createdArtifacts.clear();
|
||||
trans = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!CollectionUtils.isEmpty(createdArtifacts)) {
|
||||
tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
|
||||
}
|
||||
}
|
||||
List<CTCloudBean> repResult = getHashLookupResults(md5Hashes);
|
||||
Map<Boolean, List<CTCloudBean>> partitioned = repResult.stream()
|
||||
.filter(bean -> bean.getMalwareResult() != null)
|
||||
.collect(Collectors.partitioningBy(bean -> bean.getMalwareResult().getStatus() == Status.FOUND));
|
||||
|
||||
// TODO handle caching list and creating new items
|
||||
|
||||
createArtifacts(repResult, md5ToObjId);
|
||||
} catch (Exception ex) {
|
||||
notifyWarning(
|
||||
Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
|
||||
@ -368,6 +325,65 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
}
|
||||
}
|
||||
|
||||
private void createArtifacts(List<CTCloudBean> repResult, Map<String, List<Long>> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException {
|
||||
List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(repResult)) {
|
||||
SleuthkitCase.CaseDbTransaction trans = null;
|
||||
try {
|
||||
trans = tskCase.beginTransaction();
|
||||
for (CTCloudBean result : repResult) {
|
||||
String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue());
|
||||
List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
|
||||
if (objIds == null || objIds.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (Long objId : objIds) {
|
||||
AnalysisResult res = createAnalysisResult(objId, result, trans);
|
||||
if (res != null) {
|
||||
createdArtifacts.add(res);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trans.commit();
|
||||
trans = null;
|
||||
} finally {
|
||||
if (trans != null) {
|
||||
trans.rollback();
|
||||
createdArtifacts.clear();
|
||||
trans = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!CollectionUtils.isEmpty(createdArtifacts)) {
|
||||
tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<CTCloudBean> getHashLookupResults(List<String> md5Hashes) throws CTCloudException {
|
||||
// get an auth token with the license
|
||||
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense());
|
||||
|
||||
// make sure we are in bounds for the remaining scans
|
||||
long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
|
||||
if (remainingScans <= 0) {
|
||||
runState = RunState.DISABLED;
|
||||
notifyWarning(
|
||||
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
|
||||
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
|
||||
null);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// using auth token, get results
|
||||
return ctApiDAO.getReputationResults(
|
||||
new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
|
||||
md5Hashes
|
||||
);
|
||||
}
|
||||
|
||||
private String sanitizedMd5(String orig) {
|
||||
return StringUtils.defaultString(orig).trim().toLowerCase();
|
||||
}
|
||||
@ -392,7 +408,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
return false;
|
||||
}
|
||||
|
||||
AbstractFile af = skCase.getAbstractFileById(objId);
|
||||
AbstractFile af = tskCase.getAbstractFileById(objId);
|
||||
if (af == null) {
|
||||
return false;
|
||||
}
|
||||
@ -402,7 +418,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
}
|
||||
|
||||
// get auth token / file upload url
|
||||
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
|
||||
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense(), true);
|
||||
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
|
||||
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
|
||||
} else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
|
||||
@ -425,27 +441,29 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
.setSha1(af.getSha1Hash())
|
||||
.setSha256(af.getSha256Hash());
|
||||
|
||||
ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
|
||||
ctApiDAO.uploadMeta(new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), metaRequest);
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) {
|
||||
private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) throws InterruptedException, CTCloudException, Blackboard.BlackboardException, TskCoreException {
|
||||
// TODO integrate this
|
||||
Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping);
|
||||
|
||||
for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) {
|
||||
List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE);
|
||||
for (List<String> batch : md5Batches) {
|
||||
// TODO query and capture still unknown
|
||||
List<CTCloudBean> repResult = getHashLookupResults(batch);
|
||||
createArtifacts(repResult, remaining);
|
||||
}
|
||||
|
||||
|
||||
if (remaining.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@Messages({
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user