mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 18:17:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

work in ingest module

Browse Source
This commit is contained in:
Greg DiCristofaro 2023-07-26 08:36:03 -04:00
parent 032993c858
commit 5128468a9e
1 changed files with 79 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
View File

@ -31,6 +31,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence; import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -100,6 +101,9 @@ public class MalwareScanIngestModule implements FileIngestModule {
//minimum lookups left before issuing warning //minimum lookups left before issuing warning
private static final long LOW_LOOKUPS_REMAINING = 250; private static final long LOW_LOOKUPS_REMAINING = 250;
//minimum file uploads left before issuing warning
private static final long LOW_UPLOADS_REMAINING = 25;
private static final long MIN_UPLOAD_SIZE = 1; private static final long MIN_UPLOAD_SIZE = 1;
private static final long MAX_UPLOAD_SIZE = 1_000_000_000; private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5; private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5;
@ -123,8 +127,8 @@ public class MalwareScanIngestModule implements FileIngestModule {
private static final String MALWARE_CONFIG = "Cyber Triage Cloud"; private static final String MALWARE_CONFIG = "Cyber Triage Cloud";
private static final Logger logger = Logger.getLogger(MalwareScanIngestModule.class.getName()); private static final Logger logger = Logger.getLogger(MalwareScanIngestModule.class.getName());
private final BatchProcessor<FileRecord> batchProcessor = new BatchProcessor<FileRecord>(BATCH_SIZE, FLUSH_SECS_TIMEOUT, this::handleBatch);
private final BatchProcessor<FileRecord> batchProcessor = new BatchProcessor<FileRecord>(BATCH_SIZE, FLUSH_SECS_TIMEOUT, this::handleBatch);
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance(); private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance(); private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
@ -137,19 +141,24 @@ public class MalwareScanIngestModule implements FileIngestModule {
private BlackboardArtifact.Type malwareType = null; private BlackboardArtifact.Type malwareType = null;
private long dsId = 0; private long dsId = 0;
private long ingestJobId = 0; private long ingestJobId = 0;
private boolean uploadUnknownFiles = false; private boolean uploadUnknownFiles = false;
private Map<String, List<Long>> unidentifiedHashes = null; private Map<String, List<Long>> unidentifiedHashes = null;
@Messages({ @Messages({
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
"# {0} - remainingLookups",
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_desc=This license only has {0} lookups remaining",
"MalwareScanIngestModule_malwareTypeDisplayName=Malware", "MalwareScanIngestModule_malwareTypeDisplayName=Malware",
"MalwareScanIngestModule_ShareProcessing_noLicense_title=No Cyber Triage License", "MalwareScanIngestModule_ShareProcessing_noLicense_title=No Cyber Triage License",
"MalwareScanIngestModule_ShareProcessing_noLicense_desc=No Cyber Triage license could be loaded. Cyber Triage processing will be disabled.", "MalwareScanIngestModule_ShareProcessing_noLicense_desc=No Cyber Triage license could be loaded. Cyber Triage processing will be disabled.",
"MalwareScanIngestModule_ShareProcessing_noRemaining_title=No remaining lookups", "MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_title=No remaining lookups",
"MalwareScanIngestModule_ShareProcessing_noRemaining_desc=There are no more remaining hash lookups for this license at this time. Cyber Triage processing will be disabled." "MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_desc=There are no more remaining hash lookups for this license at this time. Cyber Triage processing will be disabled.",
}) "MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_title=Hash Lookups Low",
"# {0} - remainingLookups",
"MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_desc=This license only has {0} lookups remaining.",
"MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_title=No remaining file uploads",
"MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_desc=There are no more remaining file uploads for this license at this time. File uploading will be disabled.",
"MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_title=File Uploads Limit Low",
"# {0} - remainingUploads",
"MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_desc=This license only has {0} file uploads remaining.",})
synchronized void startUp(IngestJobContext context) throws IngestModuleException { synchronized void startUp(IngestJobContext context) throws IngestModuleException {
// only run this code once per startup // only run this code once per startup
if (runState == RunState.STARTED_UP || runState == RunState.DISABLED) { if (runState == RunState.STARTED_UP || runState == RunState.DISABLED) {
@ -175,18 +184,36 @@ public class MalwareScanIngestModule implements FileIngestModule {
long lookupsRemaining = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount()); long lookupsRemaining = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
if (lookupsRemaining <= 0) { if (lookupsRemaining <= 0) {
notifyWarning( notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_noRemaining_title(), Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_noRemaining_desc(), Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_desc(),
null); null);
runState = RunState.DISABLED; runState = RunState.DISABLED;
return; return;
} else if (lookupsRemaining < LOW_LOOKUPS_REMAINING) { } else if (lookupsRemaining < LOW_LOOKUPS_REMAINING) {
notifyWarning( notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title(), Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_lowLimitWarning_desc(lookupsRemaining), Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_desc(lookupsRemaining),
null); null);
} }
// determine lookups remaining
boolean uploadFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
if (uploadFiles) {
long uploadsRemaining = remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount());
if (uploadsRemaining <= 0) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_desc(),
null);
uploadFiles = false;
} else if (lookupsRemaining < LOW_UPLOADS_REMAINING) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_title(),
Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_desc(lookupsRemaining),
null);
}
}
// setup necessary variables for processing // setup necessary variables for processing
tskCase = Case.getCurrentCaseThrows().getSleuthkitCase(); tskCase = Case.getCurrentCaseThrows().getSleuthkitCase();
malwareType = tskCase.getBlackboard().getOrAddArtifactType( malwareType = tskCase.getBlackboard().getOrAddArtifactType(
@ -197,7 +224,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
dsId = context.getDataSource().getId(); dsId = context.getDataSource().getId();
ingestJobId = context.getJobId(); ingestJobId = context.getJobId();
licenseInfo = licenseInfoOpt.get(); licenseInfo = licenseInfoOpt.get();
uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles(); uploadUnknownFiles = uploadFiles;
unidentifiedHashes = new HashMap<>(); unidentifiedHashes = new HashMap<>();
// set run state to initialized // set run state to initialized
@ -310,13 +337,28 @@ public class MalwareScanIngestModule implements FileIngestModule {
try { try {
List<CTCloudBean> repResult = getHashLookupResults(md5Hashes); List<CTCloudBean> repResult = getHashLookupResults(md5Hashes);
Map<Boolean, List<CTCloudBean>> partitioned = repResult.stream() Map<Status, List<CTCloudBean>> statusGroupings = repResult.stream()
.filter(bean -> bean.getMalwareResult() != null) .filter(bean -> bean.getMalwareResult() != null)
.collect(Collectors.partitioningBy(bean -> bean.getMalwareResult().getStatus() == Status.FOUND)); .collect(Collectors.groupingBy(bean -> bean.getMalwareResult().getStatus()));
List<CTCloudBean> found = statusGroupings.get(Status.FOUND);
createArtifacts(repResult, md5ToObjId);
// if being scanned, check list to run later
List<CTCloudBean> beingScannedList = statusGroupings.get(Status.BEING_SCANNED);
// if not found, try upload
List<CTCloudBean> notFound = statusGroupings.get(Status.NOT_FOUND);
if (CollectionUtils.isNotEmpty(statusGroupings.get(Status.ERROR))) {
}
if (CollectionUtils.isNotEmpty(statusGroupings.get(Status.LIMITS_EXCEEDED))) {
}
// TODO handle caching list and creating new items // TODO handle caching list and creating new items
createArtifacts(repResult, md5ToObjId);
} catch (Exception ex) { } catch (Exception ex) {
notifyWarning( notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(), Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
@ -325,6 +367,25 @@ public class MalwareScanIngestModule implements FileIngestModule {
} }
} }
private void processMissing(Collection<CTCloudBean> results, Map<String, List<Long>> md5ToObjId, boolean doFileUpload) throws CTCloudException {
for (CTCloudBean beingScanned : CollectionUtils.emptyIfNull(results)) {
String sanitizedMd5 = sanitizedMd5(beingScanned.getMd5HashValue());
if (StringUtils.isBlank(sanitizedMd5)) {
continue;
}
List<Long> correspondingObjIds = md5ToObjId.get(sanitizedMd5);
if (CollectionUtils.isEmpty(correspondingObjIds)) {
continue;
}
if (doFileUpload) {
uploadFile(beingScanned, correspondingObjIds.get(0));
}
this.unidentifiedHashes.put(sanitizedMd5, correspondingObjIds);
}
}
private void createArtifacts(List<CTCloudBean> repResult, Map<String, List<Long>> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException { private void createArtifacts(List<CTCloudBean> repResult, Map<String, List<Long>> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException {
List<BlackboardArtifact> createdArtifacts = new ArrayList<>(); List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
if (!CollectionUtils.isEmpty(repResult)) { if (!CollectionUtils.isEmpty(repResult)) {