diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java index 0fa9feda54..b5d62f91de 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java @@ -62,6 +62,7 @@ import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.SystemDefaultCredentialsProvider; import org.apache.http.impl.client.WinHttpClients; +import org.sleuthkit.autopsy.coreutils.Version; /** * Actually makes the http requests to CT cloud. @@ -70,6 +71,7 @@ public class CTCloudHttpClient { private static final CTCloudHttpClient instance = new CTCloudHttpClient(); private static final Logger LOGGER = Logger.getLogger(CTCloudHttpClient.class.getName()); + private static final String HOST_URL = Version.getBuildType() == Version.Type.RELEASE ? Constants.CT_CLOUD_SERVER : Constants.CT_CLOUD_DEV_SERVER; private static final List DEFAULT_SCHEME_PRIORITY = new ArrayList<>(Arrays.asList( @@ -129,7 +131,7 @@ public class CTCloudHttpClient { } public O doPost(String urlPath, Map urlReqParams, Object jsonBody, Class classType) throws CTCloudException { - String url = Constants.CT_CLOUD_SERVER + urlPath; + String url = HOST_URL + urlPath; try { LOGGER.log(Level.INFO, "initiating http connection to ctcloud server"); diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/Constants.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/Constants.java index fa35dd7dbb..ddda5b4508 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/Constants.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/Constants.java @@ -34,8 +34,7 @@ final public class Constants { public static final String CT_CLOUD_DEV_SERVER = "https://cyber-triage-dev.appspot.com"; - // TODO put back - public static final String CT_CLOUD_SERVER = CT_CLOUD_DEV_SERVER; //"https://rep1.cybertriage.com"; + public static final String CT_CLOUD_SERVER = "https://rep1.cybertriage.com"; /** * Link to watch demo video diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties index 68677cb3f7..46f62fdf7a 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties @@ -1,8 +1,8 @@ # Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license # Click nbfs://nbhost/SystemFileSystem/Templates/Other/properties.properties to edit this template -OptionsCategory_Name_CyberTriage=CyberTriage -OptionsCategory_Keywords_CyberTriage=CyberTriage,Cyber,Triage +OptionsCategory_Name_CyberTriage=Cyber Triage +OptionsCategory_Keywords_CyberTriage=Cyber Triage,Cyber,Triage LicenseDisclaimerPanel.disclaimer.text=The Cyber Triage Malware Scanner module uses 40+ malware scanning engines to identify if Windows executables are malicious. It requires a non-free license to use. LicenseDisclaimerPanel.purchaseFromLabel.text=You can purchase a license from LicenseDisclaimerPanel.link.text=https://cybertriage.com/autopsy-checkout diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties-MERGED b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties-MERGED index 68677cb3f7..46f62fdf7a 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties-MERGED +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties-MERGED @@ -1,8 +1,8 @@ # Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license # Click nbfs://nbhost/SystemFileSystem/Templates/Other/properties.properties to edit this template -OptionsCategory_Name_CyberTriage=CyberTriage -OptionsCategory_Keywords_CyberTriage=CyberTriage,Cyber,Triage +OptionsCategory_Name_CyberTriage=Cyber Triage +OptionsCategory_Keywords_CyberTriage=Cyber Triage,Cyber,Triage LicenseDisclaimerPanel.disclaimer.text=The Cyber Triage Malware Scanner module uses 40+ malware scanning engines to identify if Windows executables are malicious. It requires a non-free license to use. LicenseDisclaimerPanel.purchaseFromLabel.text=You can purchase a license from LicenseDisclaimerPanel.link.text=https://cybertriage.com/autopsy-checkout diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/LicenseDisclaimerPanel.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/LicenseDisclaimerPanel.java index ffd3760e6f..4299f02d62 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/LicenseDisclaimerPanel.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/LicenseDisclaimerPanel.java @@ -22,16 +22,18 @@ import java.awt.Desktop; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; /** * Disclaimer for license and place to purchase CT license. */ public class LicenseDisclaimerPanel extends javax.swing.JPanel { + private static final Logger LOGGER = Logger.getLogger(LicenseDisclaimerPanel.class.getName()); - + private static final String CHECKOUT_PAGE_URL = "https://cybertriage.com/autopsy-checkout"; - + /** * Creates new form LicenseDisclaimerPanel */ @@ -116,9 +118,10 @@ public class LicenseDisclaimerPanel extends javax.swing.JPanel { try { Desktop.getDesktop().browse(new URI(CHECKOUT_PAGE_URL)); } catch (IOException | URISyntaxException e) { - /* TODO: error handling */ } + LOGGER.log(Level.SEVERE, "Error opening link to: " + CHECKOUT_PAGE_URL, e); + } } else { - /* TODO: error handling */ + LOGGER.log(Level.WARNING, "Desktop API is not supported. Link cannot be opened."); } }//GEN-LAST:event_linkMouseClicked diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicenseDialog.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicenseDialog.java index 11af572890..608ea63040 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicenseDialog.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicenseDialog.java @@ -29,9 +29,7 @@ import org.openide.util.NbBundle.Messages; */ public class CTLicenseDialog extends javax.swing.JDialog { - //private static final Pattern LICENSE_PATTERN = Pattern.compile("^\\s*[0-9a-zA-Z]{8}\\-[0-9a-zA-Z]{4}\\-[0-9a-zA-Z]{4}\\-[0-9a-zA-Z]{4}\\-[0-9a-zA-Z]{12}\\s*$"); - - private static final Pattern LICENSE_PATTERN = Pattern.compile("\\s*[a-zA-Z0-9\\-]+?\\s*"); + private static final Pattern LICENSE_PATTERN = Pattern.compile("^\\s*[a-zA-Z0-9\\-]+?\\s*$"); private String licenseString = null; /** diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/BatchProcessor.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/BatchProcessor.java index dc62f6a05e..eab025a641 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/BatchProcessor.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/BatchProcessor.java @@ -38,27 +38,18 @@ import java.util.function.Consumer; */ public class BatchProcessor { - private final ExecutorService processingExecutorService = Executors.newSingleThreadExecutor(); + private ExecutorService processingExecutorService = Executors.newSingleThreadExecutor(); private final BlockingQueue batchingQueue; - private final List processingQueue; private final int batchSize; private final Consumer> itemsConsumer; - private final long millisTimeout; + private final long secondsTimeout; - private Future lastProcessingFuture = CompletableFuture.runAsync(() -> { - }); - - public BatchProcessor(int batchSize, long millisTimeout, Consumer> itemsConsumer) { + public BatchProcessor(int batchSize, long secondsTimeout, Consumer> itemsConsumer) { this.batchingQueue = new LinkedBlockingQueue<>(batchSize); - this.processingQueue = new ArrayList<>(batchSize); this.batchSize = batchSize; this.itemsConsumer = itemsConsumer; - this.millisTimeout = millisTimeout; - } - - public synchronized void clearCurrentBatch() { - batchingQueue.clear(); + this.secondsTimeout = secondsTimeout; } public synchronized void add(T item) throws InterruptedException { @@ -68,40 +59,29 @@ public class BatchProcessor { } } - public synchronized void flush(boolean blockUntilFinished) throws InterruptedException { + public synchronized void flushAndReset() throws InterruptedException { + // get any remaining asyncProcessBatch(); - if (blockUntilFinished) { - waitCurrentFuture(); - } - } - - private synchronized void waitCurrentFuture() throws InterruptedException { - synchronized (lastProcessingFuture) { - if (!lastProcessingFuture.isDone()) { - try { - lastProcessingFuture.get(millisTimeout, TimeUnit.MILLISECONDS); - } catch (ExecutionException | TimeoutException ex) { - // ignore timeout - } - } - } + + // don't accept any new additions + processingExecutorService.shutdown(); + + // await termination + processingExecutorService.awaitTermination(secondsTimeout, TimeUnit.SECONDS); + + // get new (not shut down executor) + processingExecutorService = Executors.newSingleThreadExecutor(); } private synchronized void asyncProcessBatch() throws InterruptedException { if (!batchingQueue.isEmpty()) { - // wait for previous processing to finish - waitCurrentFuture(); - - // if 'andThen' doesn't run, clear the processing queue - processingQueue.clear(); + final List processingList = new ArrayList<>(); // transfer batching queue to processing queue - batchingQueue.drainTo(processingQueue); + batchingQueue.drainTo(processingList); - // submit to processor and then clear processing queue - lastProcessingFuture = processingExecutorService.submit( - () -> itemsConsumer.andThen(processingQueue -> processingQueue.clear()).accept(processingQueue) - ); + // submit to be processed + processingExecutorService.submit(() -> itemsConsumer.accept(processingList)); } } diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java index a509310391..e249b487d5 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java @@ -28,6 +28,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo; import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean; import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest; import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -54,6 +55,9 @@ import org.sleuthkit.datamodel.AnalysisResult; import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.ReadContentInputStream; +import org.sleuthkit.datamodel.HashUtility; +import org.sleuthkit.datamodel.HashUtility.HashResult; +import org.sleuthkit.datamodel.HashUtility.HashType; import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -89,8 +93,8 @@ public class MalwareScanIngestModule implements FileIngestModule { // batch size of 200 files max private static final int BATCH_SIZE = 200; - // 3 minute timeout for an API request - private static final long BATCH_MILLIS_TIMEOUT = 3 * 60 * 1000; + // 1 day timeout for all API requests + private static final long FLUSH_SECS_TIMEOUT = 24 * 60 * 60; //minimum lookups left before issuing warning private static final long LOW_LOOKUPS_REMAINING = 250; @@ -119,7 +123,7 @@ public class MalwareScanIngestModule implements FileIngestModule { private static final String MALWARE_CONFIG = "Cyber Triage Cloud"; private static final Logger logger = Logger.getLogger(MalwareScanIngestModule.class.getName()); - private final BatchProcessor batchProcessor = new BatchProcessor(BATCH_SIZE, BATCH_MILLIS_TIMEOUT, this::handleBatch); + private final BatchProcessor batchProcessor = new BatchProcessor(BATCH_SIZE, FLUSH_SECS_TIMEOUT, this::handleBatch); private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance(); private final CTApiDAO ctApiDAO = CTApiDAO.getInstance(); @@ -207,18 +211,54 @@ public class MalwareScanIngestModule implements FileIngestModule { return limit - used; } + private String getOrCalcHash(AbstractFile af) { + if (StringUtils.isNotBlank(af.getMd5Hash())) { + return af.getMd5Hash(); + } + + try { + List hashResults = HashUtility.calculateHashes(af, Collections.singletonList(HashType.MD5)); + if (CollectionUtils.isNotEmpty(hashResults)) { + for (HashResult hashResult : hashResults) { + if (hashResult.getType() == HashType.MD5) { + return hashResult.getValue(); + } + } + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, + MessageFormat.format("An error occurred while processing file name: {0} and obj id: {1}.", + af.getName(), + af.getId()), + ex); + } + + return null; + } + @Messages({ "MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout", "MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out" }) IngestModule.ProcessResult process(AbstractFile af) { try { - if (runState == RunState.STARTED_UP && af.getKnown() != TskData.FileKnown.KNOWN - && EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(fileTypeDetector.getMIMEType(af)).trim().toLowerCase())) { - batchProcessor.add(new FileRecord(af.getId(), af.getMd5Hash())); + if (runState == RunState.STARTED_UP + && af.getKnown() != TskData.FileKnown.KNOWN + && EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(fileTypeDetector.getMIMEType(af)).trim().toLowerCase()) + && CollectionUtils.isEmpty(af.getAnalysisResults(malwareType))) { + String md5 = getOrCalcHash(af); + if (StringUtils.isNotBlank(md5)) { + batchProcessor.add(new FileRecord(af.getId(), md5)); + } } return ProcessResult.OK; + } catch (TskCoreException ex) { + notifyWarning( + Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(), + Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(), + ex); + return IngestModule.ProcessResult.ERROR; } catch (InterruptedException ex) { notifyWarning( Bundle.MalwareScanIngestModule_ShareProcessing_batchTimeout_title(), @@ -246,7 +286,7 @@ public class MalwareScanIngestModule implements FileIngestModule { // create mapping of md5 to corresponding object ids as well as just the list of md5's Map> md5ToObjId = new HashMap<>(); - List md5Hashes = new ArrayList<>(); + for (FileRecord fr : fileRecords) { if (fr == null || StringUtils.isBlank(fr.getMd5hash()) || fr.getObjId() <= 0) { continue; @@ -257,9 +297,10 @@ public class MalwareScanIngestModule implements FileIngestModule { .computeIfAbsent(sanitizedMd5, (k) -> new ArrayList<>()) .add(fr.getObjId()); - md5Hashes.add(sanitizedMd5); } + List md5Hashes = new ArrayList<>(md5ToObjId.keySet()); + if (md5Hashes.isEmpty()) { return; } @@ -279,13 +320,6 @@ public class MalwareScanIngestModule implements FileIngestModule { return; } - // if the size of this batch will exceed limit, shrink list to limit and fail after processing - boolean exceededScanLimit = false; - if (remainingScans < md5Hashes.size()) { - md5Hashes = md5Hashes.subList(0, (int) remainingScans); - exceededScanLimit = true; - } - // using auth token, get results List repResult = ctApiDAO.getReputationResults( new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse), @@ -325,16 +359,6 @@ public class MalwareScanIngestModule implements FileIngestModule { if (!CollectionUtils.isEmpty(createdArtifacts)) { tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId); } - - // if we only processed part of the batch, after processing, notify that we are out of scans. - if (exceededScanLimit) { - runState = RunState.DISABLED; - notifyWarning( - Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(), - Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(), - null); - return; - } } } catch (Exception ex) { notifyWarning( @@ -467,7 +491,7 @@ public class MalwareScanIngestModule implements FileIngestModule { // flush any remaining items try { - batchProcessor.flush(true); + batchProcessor.flushAndReset(); } catch (InterruptedException ex) { notifyWarning( Bundle.MalwareScanIngestModule_SharedProcessing_flushTimeout_title(), @@ -476,7 +500,6 @@ public class MalwareScanIngestModule implements FileIngestModule { } finally { // set state to shut down and clear any remaining runState = RunState.SHUT_DOWN; - batchProcessor.clearCurrentBatch(); } }