mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Greg DiCristofaro 2023-07-25 19:07:32 -04:00
parent f23331b64c
commit 032993c858
1 changed files with 82 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
View File

@ -26,6 +26,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean;
import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse;
import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo; import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean; import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest; import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence; import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
import java.text.MessageFormat; import java.text.MessageFormat;
@ -103,7 +104,6 @@ public class MalwareScanIngestModule implements FileIngestModule {
private static final long MAX_UPLOAD_SIZE = 1_000_000_000; private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5; private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5;
private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000; private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of( private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
"application/x-bat",//NON-NLS "application/x-bat",//NON-NLS
@ -128,6 +128,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance(); private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance(); private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
// TODO minimize state
private RunState runState = null; private RunState runState = null;
private SleuthkitCase tskCase = null; private SleuthkitCase tskCase = null;
@ -137,6 +138,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
private long dsId = 0; private long dsId = 0;
private long ingestJobId = 0; private long ingestJobId = 0;
private boolean uploadUnknownFiles = false; private boolean uploadUnknownFiles = false;
private Map<String, List<Long>> unidentifiedHashes = null;
@Messages({ @Messages({
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low", "MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
@ -196,7 +198,8 @@ public class MalwareScanIngestModule implements FileIngestModule {
ingestJobId = context.getJobId(); ingestJobId = context.getJobId();
licenseInfo = licenseInfoOpt.get(); licenseInfo = licenseInfoOpt.get();
uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles(); uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
unidentifiedHashes = new HashMap<>();
// set run state to initialized // set run state to initialized
runState = RunState.STARTED_UP; runState = RunState.STARTED_UP;
} catch (Exception ex) { } catch (Exception ex) {
@ -306,60 +309,14 @@ public class MalwareScanIngestModule implements FileIngestModule {
} }
try { try {
// get an auth token with the license List<CTCloudBean> repResult = getHashLookupResults(md5Hashes);
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense()); Map<Boolean, List<CTCloudBean>> partitioned = repResult.stream()
.filter(bean -> bean.getMalwareResult() != null)
// make sure we are in bounds for the remaining scans .collect(Collectors.partitioningBy(bean -> bean.getMalwareResult().getStatus() == Status.FOUND));
long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
if (remainingScans <= 0) { // TODO handle caching list and creating new items
runState = RunState.DISABLED;
notifyWarning( createArtifacts(repResult, md5ToObjId);
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
null);
return;
}
// using auth token, get results
List<CTCloudBean> repResult = ctApiDAO.getReputationResults(
new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
md5Hashes
);
List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
if (!CollectionUtils.isEmpty(repResult)) {
SleuthkitCase.CaseDbTransaction trans = null;
try {
trans = tskCase.beginTransaction();
for (CTCloudBean result : repResult) {
String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue());
List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
if (objIds == null || objIds.isEmpty()) {
continue;
}
for (Long objId : objIds) {
AnalysisResult res = createAnalysisResult(objId, result, trans);
if (res != null) {
createdArtifacts.add(res);
}
}
}
trans.commit();
trans = null;
} finally {
if (trans != null) {
trans.rollback();
createdArtifacts.clear();
trans = null;
}
}
if (!CollectionUtils.isEmpty(createdArtifacts)) {
tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
}
}
} catch (Exception ex) { } catch (Exception ex) {
notifyWarning( notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(), Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
@ -368,6 +325,65 @@ public class MalwareScanIngestModule implements FileIngestModule {
} }
} }
private void createArtifacts(List<CTCloudBean> repResult, Map<String, List<Long>> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException {
List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
if (!CollectionUtils.isEmpty(repResult)) {
SleuthkitCase.CaseDbTransaction trans = null;
try {
trans = tskCase.beginTransaction();
for (CTCloudBean result : repResult) {
String sanitizedMd5 = sanitizedMd5(result.getMd5HashValue());
List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
if (objIds == null || objIds.isEmpty()) {
continue;
}
for (Long objId : objIds) {
AnalysisResult res = createAnalysisResult(objId, result, trans);
if (res != null) {
createdArtifacts.add(res);
}
}
}
trans.commit();
trans = null;
} finally {
if (trans != null) {
trans.rollback();
createdArtifacts.clear();
trans = null;
}
}
if (!CollectionUtils.isEmpty(createdArtifacts)) {
tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
}
}
}
private List<CTCloudBean> getHashLookupResults(List<String> md5Hashes) throws CTCloudException {
// get an auth token with the license
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense());
// make sure we are in bounds for the remaining scans
long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
if (remainingScans <= 0) {
runState = RunState.DISABLED;
notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
null);
return Collections.emptyList();
}
// using auth token, get results
return ctApiDAO.getReputationResults(
new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
md5Hashes
);
}
private String sanitizedMd5(String orig) { private String sanitizedMd5(String orig) {
return StringUtils.defaultString(orig).trim().toLowerCase(); return StringUtils.defaultString(orig).trim().toLowerCase();
} }
@ -392,7 +408,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
return false; return false;
} }
AbstractFile af = skCase.getAbstractFileById(objId); AbstractFile af = tskCase.getAbstractFileById(objId);
if (af == null) { if (af == null) {
return false; return false;
} }
@ -402,7 +418,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
} }
// get auth token / file upload url // get auth token / file upload url
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true); AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfo.getDecryptedLicense(), true);
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) { if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR); throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
} else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) { } else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
@ -425,27 +441,29 @@ public class MalwareScanIngestModule implements FileIngestModule {
.setSha1(af.getSha1Hash()) .setSha1(af.getSha1Hash())
.setSha256(af.getSha256Hash()); .setSha256(af.getSha256Hash());
ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest); ctApiDAO.uploadMeta(new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), metaRequest);
return true; return true;
} }
private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) { private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) throws InterruptedException, CTCloudException, Blackboard.BlackboardException, TskCoreException {
// TODO integrate this
Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping); Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping);
for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) { for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) {
List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE); List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE);
for (List<String> batch : md5Batches) { for (List<String> batch : md5Batches) {
// TODO query and capture still unknown List<CTCloudBean> repResult = getHashLookupResults(batch);
createArtifacts(repResult, remaining);
} }
if (remaining.isEmpty()) { if (remaining.isEmpty()) {
return true; return true;
} }
Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS); Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
} }
return false;
} }
@Messages({ @Messages({