mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge from 7160

Browse Source
This commit is contained in:
Greg DiCristofaro 2023-07-25 16:58:34 -04:00
commit f23331b64c
8 changed files with 85 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
View File

@ -62,6 +62,7 @@ import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.SystemDefaultCredentialsProvider;
import org.apache.http.impl.client.WinHttpClients;
import org.sleuthkit.autopsy.coreutils.Version;
/**
* Actually makes the http requests to CT cloud.
@ -70,6 +71,7 @@ public class CTCloudHttpClient {
private static final CTCloudHttpClient instance = new CTCloudHttpClient();
private static final Logger LOGGER = Logger.getLogger(CTCloudHttpClient.class.getName());
private static final String HOST_URL = Version.getBuildType() == Version.Type.RELEASE ? Constants.CT_CLOUD_SERVER : Constants.CT_CLOUD_DEV_SERVER;
private static final List<String> DEFAULT_SCHEME_PRIORITY
= new ArrayList<>(Arrays.asList(
@ -129,7 +131,7 @@ public class CTCloudHttpClient {
}
public <O> O doPost(String urlPath, Map<String, String> urlReqParams, Object jsonBody, Class<O> classType) throws CTCloudException {
String url = Constants.CT_CLOUD_SERVER + urlPath;
String url = HOST_URL + urlPath;
try {
LOGGER.log(Level.INFO, "initiating http connection to ctcloud server");

3
Core/src/com/basistech/df/cybertriage/autopsy/ctapi/Constants.java
View File

@ -34,8 +34,7 @@ final public class Constants {
public static final String CT_CLOUD_DEV_SERVER = "https://cyber-triage-dev.appspot.com";
// TODO put back
public static final String CT_CLOUD_SERVER = CT_CLOUD_DEV_SERVER; //"https://rep1.cybertriage.com";
public static final String CT_CLOUD_SERVER = "https://rep1.cybertriage.com";
/**
* Link to watch demo video

4
Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties
View File

@ -1,8 +1,8 @@
# Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
# Click nbfs://nbhost/SystemFileSystem/Templates/Other/properties.properties to edit this template
OptionsCategory_Name_CyberTriage=CyberTriage
OptionsCategory_Keywords_CyberTriage=CyberTriage,Cyber,Triage
OptionsCategory_Name_CyberTriage=Cyber Triage
OptionsCategory_Keywords_CyberTriage=Cyber Triage,Cyber,Triage
LicenseDisclaimerPanel.disclaimer.text=<html>The Cyber Triage Malware Scanner module uses 40+ malware scanning engines to identify if Windows executables are malicious. It requires a non-free license to use.</html>
LicenseDisclaimerPanel.purchaseFromLabel.text=You can purchase a license from
LicenseDisclaimerPanel.link.text=<html><span style="color: blue; text-decoration: underline">https://cybertriage.com/autopsy-checkout</span></html>

4
Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties-MERGED
View File

@ -1,8 +1,8 @@
# Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
# Click nbfs://nbhost/SystemFileSystem/Templates/Other/properties.properties to edit this template
OptionsCategory_Name_CyberTriage=CyberTriage
OptionsCategory_Keywords_CyberTriage=CyberTriage,Cyber,Triage
OptionsCategory_Name_CyberTriage=Cyber Triage
OptionsCategory_Keywords_CyberTriage=Cyber Triage,Cyber,Triage
LicenseDisclaimerPanel.disclaimer.text=<html>The Cyber Triage Malware Scanner module uses 40+ malware scanning engines to identify if Windows executables are malicious. It requires a non-free license to use.</html>
LicenseDisclaimerPanel.purchaseFromLabel.text=You can purchase a license from
LicenseDisclaimerPanel.link.text=<html><span style="color: blue; text-decoration: underline">https://cybertriage.com/autopsy-checkout</span></html>

7
Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/LicenseDisclaimerPanel.java
View File

@ -22,12 +22,14 @@ import java.awt.Desktop;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
/**
* Disclaimer for license and place to purchase CT license.
*/
public class LicenseDisclaimerPanel extends javax.swing.JPanel {
private static final Logger LOGGER = Logger.getLogger(LicenseDisclaimerPanel.class.getName());
private static final String CHECKOUT_PAGE_URL = "https://cybertriage.com/autopsy-checkout";
@ -116,9 +118,10 @@ public class LicenseDisclaimerPanel extends javax.swing.JPanel {
try {
Desktop.getDesktop().browse(new URI(CHECKOUT_PAGE_URL));
} catch (IOException | URISyntaxException e) {
/* TODO: error handling */ }
LOGGER.log(Level.SEVERE, "Error opening link to: " + CHECKOUT_PAGE_URL, e);
}
} else {
/* TODO: error handling */
LOGGER.log(Level.WARNING, "Desktop API is not supported. Link cannot be opened.");
}
}//GEN-LAST:event_linkMouseClicked

4
Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicenseDialog.java
View File

@ -29,9 +29,7 @@ import org.openide.util.NbBundle.Messages;
*/
public class CTLicenseDialog extends javax.swing.JDialog {
//private static final Pattern LICENSE_PATTERN = Pattern.compile("^\\s*[0-9a-zA-Z]{8}\\-[0-9a-zA-Z]{4}\\-[0-9a-zA-Z]{4}\\-[0-9a-zA-Z]{4}\\-[0-9a-zA-Z]{12}\\s*$");
private static final Pattern LICENSE_PATTERN = Pattern.compile("\\s*[a-zA-Z0-9\\-]+?\\s*");
private static final Pattern LICENSE_PATTERN = Pattern.compile("^\\s*[a-zA-Z0-9\\-]+?\\s*$");
private String licenseString = null;
/**

56
Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/BatchProcessor.java
View File

@ -38,27 +38,18 @@ import java.util.function.Consumer;
*/
public class BatchProcessor<T> {
private final ExecutorService processingExecutorService = Executors.newSingleThreadExecutor();
private ExecutorService processingExecutorService = Executors.newSingleThreadExecutor();
private final BlockingQueue<T> batchingQueue;
private final List<T> processingQueue;
private final int batchSize;
private final Consumer<List<T>> itemsConsumer;
private final long millisTimeout;
private final long secondsTimeout;
private Future<?> lastProcessingFuture = CompletableFuture.runAsync(() -> {
});
public BatchProcessor(int batchSize, long millisTimeout, Consumer<List<T>> itemsConsumer) {
public BatchProcessor(int batchSize, long secondsTimeout, Consumer<List<T>> itemsConsumer) {
this.batchingQueue = new LinkedBlockingQueue<>(batchSize);
this.processingQueue = new ArrayList<>(batchSize);
this.batchSize = batchSize;
this.itemsConsumer = itemsConsumer;
this.millisTimeout = millisTimeout;
}
public synchronized void clearCurrentBatch() {
batchingQueue.clear();
this.secondsTimeout = secondsTimeout;
}
public synchronized void add(T item) throws InterruptedException {
@ -68,40 +59,29 @@ public class BatchProcessor<T> {
}
}
public synchronized void flush(boolean blockUntilFinished) throws InterruptedException {
public synchronized void flushAndReset() throws InterruptedException {
// get any remaining
asyncProcessBatch();
if (blockUntilFinished) {
waitCurrentFuture();
}
}
private synchronized void waitCurrentFuture() throws InterruptedException {
synchronized (lastProcessingFuture) {
if (!lastProcessingFuture.isDone()) {
try {
lastProcessingFuture.get(millisTimeout, TimeUnit.MILLISECONDS);
} catch (ExecutionException | TimeoutException ex) {
// ignore timeout
}
}
}
// don't accept any new additions
processingExecutorService.shutdown();
// await termination
processingExecutorService.awaitTermination(secondsTimeout, TimeUnit.SECONDS);
// get new (not shut down executor)
processingExecutorService = Executors.newSingleThreadExecutor();
}
private synchronized void asyncProcessBatch() throws InterruptedException {
if (!batchingQueue.isEmpty()) {
// wait for previous processing to finish
waitCurrentFuture();
// if 'andThen' doesn't run, clear the processing queue
processingQueue.clear();
final List<T> processingList = new ArrayList<>();
// transfer batching queue to processing queue
batchingQueue.drainTo(processingQueue);
batchingQueue.drainTo(processingList);
// submit to processor and then clear processing queue
lastProcessingFuture = processingExecutorService.submit(
() -> itemsConsumer.andThen(processingQueue -> processingQueue.clear()).accept(processingQueue)
);
// submit to be processed
processingExecutorService.submit(() -> itemsConsumer.accept(processingList));
}
}

77
Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
View File

@ -28,6 +28,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -54,6 +55,9 @@ import org.sleuthkit.datamodel.AnalysisResult;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.HashUtility.HashResult;
import org.sleuthkit.datamodel.HashUtility.HashType;
import org.sleuthkit.datamodel.Score;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
@ -89,8 +93,8 @@ public class MalwareScanIngestModule implements FileIngestModule {
// batch size of 200 files max
private static final int BATCH_SIZE = 200;
// 3 minute timeout for an API request
private static final long BATCH_MILLIS_TIMEOUT = 3 * 60 * 1000;
// 1 day timeout for all API requests
private static final long FLUSH_SECS_TIMEOUT = 24 * 60 * 60;
//minimum lookups left before issuing warning
private static final long LOW_LOOKUPS_REMAINING = 250;
@ -119,7 +123,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
private static final String MALWARE_CONFIG = "Cyber Triage Cloud";
private static final Logger logger = Logger.getLogger(MalwareScanIngestModule.class.getName());
private final BatchProcessor<FileRecord> batchProcessor = new BatchProcessor<FileRecord>(BATCH_SIZE, BATCH_MILLIS_TIMEOUT, this::handleBatch);
private final BatchProcessor<FileRecord> batchProcessor = new BatchProcessor<FileRecord>(BATCH_SIZE, FLUSH_SECS_TIMEOUT, this::handleBatch);
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
@ -207,18 +211,54 @@ public class MalwareScanIngestModule implements FileIngestModule {
return limit - used;
}
private String getOrCalcHash(AbstractFile af) {
if (StringUtils.isNotBlank(af.getMd5Hash())) {
return af.getMd5Hash();
}
try {
List<HashResult> hashResults = HashUtility.calculateHashes(af, Collections.singletonList(HashType.MD5));
if (CollectionUtils.isNotEmpty(hashResults)) {
for (HashResult hashResult : hashResults) {
if (hashResult.getType() == HashType.MD5) {
return hashResult.getValue();
}
}
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING,
MessageFormat.format("An error occurred while processing file name: {0} and obj id: {1}.",
af.getName(),
af.getId()),
ex);
}
return null;
}
@Messages({
"MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout",
"MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out"
})
IngestModule.ProcessResult process(AbstractFile af) {
try {
if (runState == RunState.STARTED_UP && af.getKnown() != TskData.FileKnown.KNOWN
&& EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(fileTypeDetector.getMIMEType(af)).trim().toLowerCase())) {
batchProcessor.add(new FileRecord(af.getId(), af.getMd5Hash()));
if (runState == RunState.STARTED_UP
&& af.getKnown() != TskData.FileKnown.KNOWN
&& EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(fileTypeDetector.getMIMEType(af)).trim().toLowerCase())
&& CollectionUtils.isEmpty(af.getAnalysisResults(malwareType))) {
String md5 = getOrCalcHash(af);
if (StringUtils.isNotBlank(md5)) {
batchProcessor.add(new FileRecord(af.getId(), md5));
}
}
return ProcessResult.OK;
} catch (TskCoreException ex) {
notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
ex);
return IngestModule.ProcessResult.ERROR;
} catch (InterruptedException ex) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_batchTimeout_title(),
@ -246,7 +286,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
// create mapping of md5 to corresponding object ids as well as just the list of md5's
Map<String, List<Long>> md5ToObjId = new HashMap<>();
List<String> md5Hashes = new ArrayList<>();
for (FileRecord fr : fileRecords) {
if (fr == null || StringUtils.isBlank(fr.getMd5hash()) || fr.getObjId() <= 0) {
continue;
@ -257,9 +297,10 @@ public class MalwareScanIngestModule implements FileIngestModule {
.computeIfAbsent(sanitizedMd5, (k) -> new ArrayList<>())
.add(fr.getObjId());
md5Hashes.add(sanitizedMd5);
}
List<String> md5Hashes = new ArrayList<>(md5ToObjId.keySet());
if (md5Hashes.isEmpty()) {
return;
}
@ -279,13 +320,6 @@ public class MalwareScanIngestModule implements FileIngestModule {
return;
}
// if the size of this batch will exceed limit, shrink list to limit and fail after processing
boolean exceededScanLimit = false;
if (remainingScans < md5Hashes.size()) {
md5Hashes = md5Hashes.subList(0, (int) remainingScans);
exceededScanLimit = true;
}
// using auth token, get results
List<CTCloudBean> repResult = ctApiDAO.getReputationResults(
new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
@ -325,16 +359,6 @@ public class MalwareScanIngestModule implements FileIngestModule {
if (!CollectionUtils.isEmpty(createdArtifacts)) {
tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
}
// if we only processed part of the batch, after processing, notify that we are out of scans.
if (exceededScanLimit) {
runState = RunState.DISABLED;
notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
null);
return;
}
}
} catch (Exception ex) {
notifyWarning(
@ -467,7 +491,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
// flush any remaining items
try {
batchProcessor.flush(true);
batchProcessor.flushAndReset();
} catch (InterruptedException ex) {
notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_flushTimeout_title(),
@ -476,7 +500,6 @@ public class MalwareScanIngestModule implements FileIngestModule {
} finally {
// set state to shut down and clear any remaining
runState = RunState.SHUT_DOWN;
batchProcessor.clearCurrentBatch();
}
}