updates

Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicensePersistence.java

@@ -92,19 +92,15 @@ public class CTLicensePersistence {
         });
     }
 
-        public synchronized boolean saveMalwareSettings(MalwareIngestSettings malwareIngestSettings) {
+    public synchronized boolean saveMalwareSettings(MalwareIngestSettings malwareIngestSettings) {
         if (malwareIngestSettings != null) {
             File settingsFile = getMalwareIngestFile();
             try {
                 settingsFile.getParentFile().mkdirs();
-                if (licenseResponse != null) {
+                objectMapper.writeValue(settingsFile, malwareIngestSettings);
-                    objectMapper.writeValue(licenseFile, licenseResponse);
-                } else if (licenseFile.exists()) {
-                    Files.delete(licenseFile.toPath());
-                }
                 return true;
             } catch (IOException ex) {
-                logger.log(Level.WARNING, "There was an error writing CyberTriage license to file: " + licenseFile.getAbsolutePath(), ex);
+                logger.log(Level.WARNING, "There was an error writing malware ingest settings to file: " + settingsFile.getAbsolutePath(), ex);
             }
         }
 
@@ -112,19 +108,23 @@ public class CTLicensePersistence {
     }
 
     public synchronized MalwareIngestSettings loadMalwareIngestSettings() {
-        Optional<LicenseResponse> toRet = Optional.empty();
+        MalwareIngestSettings settings = null;
-        File licenseFile = getCTLicenseFile();
+        File settingsFile = getMalwareIngestFile();
-        if (licenseFile.exists() && licenseFile.isFile()) {
+        if (settingsFile.exists() && settingsFile.isFile()) {
             try {
-                toRet = Optional.ofNullable(objectMapper.readValue(licenseFile, LicenseResponse.class));
+                settings = objectMapper.readValue(settingsFile, MalwareIngestSettings.class);
             } catch (IOException ex) {
-                logger.log(Level.WARNING, "There was an error reading CyberTriage license to file: " + licenseFile.getAbsolutePath(), ex);
+                logger.log(Level.WARNING, "There was an error reading malware ingest settings from file: " + settingsFile.getAbsolutePath(), ex);
             }
         }
+        
+        if (settings == null) {
+            settings = new MalwareIngestSettings();
+        }
 
-        return toRet;
+        return settings;
     }
-    
+
     private File getCTLicenseFile() {
         return Paths.get(PlatformUtil.getModuleConfigDirectory(), CT_SETTINGS_DIR, CT_LICENSE_FILENAME).toFile();
     }

Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTMalwareScannerOptionsPanel.java

@@ -111,6 +111,7 @@ public class CTMalwareScannerOptionsPanel extends CTOptionsSubPanel {
     @Override
     public synchronized void saveSettings() {
         ctPersistence.saveLicenseResponse(getLicenseInfo());
+        ctPersistence.saveMalwareSettings(getIngestSettings());
     }
 
     @Override
@@ -127,11 +128,26 @@ public class CTMalwareScannerOptionsPanel extends CTOptionsSubPanel {
         if (licenseInfo != null) {
             loadMalwareScansInfo(licenseInfo);
         }
+        
+        MalwareIngestSettings ingestSettings = ctPersistence.loadMalwareIngestSettings();
+        setIngestSettings(ingestSettings);
     }
 
     private synchronized LicenseResponse getLicenseInfo() {
         return this.licenseInfo == null ? null : this.licenseInfo.getLicenseResponse();
     }
+    
+    private MalwareIngestSettings getIngestSettings() {
+        return new MalwareIngestSettings()
+                .setUploadFiles(this.fileUploadCheckbox.isSelected());
+    }
+    
+    private void setIngestSettings(MalwareIngestSettings ingestSettings) {
+        if (ingestSettings == null) {
+            ingestSettings = new MalwareIngestSettings();
+        }
+        this.fileUploadCheckbox.setSelected(ingestSettings.isUploadFiles());
+    }
 
     private synchronized void setLicenseDisplay(LicenseInfo licenseInfo, String licenseMessage) {
         this.licenseInfo = licenseInfo;
@@ -411,7 +427,7 @@ public class CTMalwareScannerOptionsPanel extends CTOptionsSubPanel {
     }//GEN-LAST:event_licenseInfoAddButtonActionPerformed
 
     private void fileUploadCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_fileUploadCheckboxActionPerformed
-        // TODO add your handling code here:
+        this.firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
     }//GEN-LAST:event_fileUploadCheckboxActionPerformed
 
     @NbBundle.Messages({

Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java

@@ -40,6 +40,7 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.shaded.com.google.common.collect.Lists;
 import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.coreutils.Logger;
@@ -96,6 +97,9 @@ public class MalwareScanIngestModule implements FileIngestModule {
 
         private static final long MIN_UPLOAD_SIZE = 1;
         private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
+        private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5;
+        private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
+        
 
         private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
                 "application/x-bat",//NON-NLS
@@ -128,6 +132,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
         private BlackboardArtifact.Type malwareType = null;
         private long dsId = 0;
         private long ingestJobId = 0;
+        private boolean uploadUnknownFiles = false;
 
         @Messages({
             "MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
@@ -186,6 +191,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
                 dsId = context.getDataSource().getId();
                 ingestJobId = context.getJobId();
                 licenseInfo = licenseInfoOpt.get();
+                uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
 
                 // set run state to initialized
                 runState = RunState.STARTED_UP;
@@ -201,55 +207,6 @@ public class MalwareScanIngestModule implements FileIngestModule {
             return limit - used;
         }
 
-        private boolean isUnknown(CTCloudBean cloudBean) {
-            return cloudBean != null
-                    && cloudBean.getMalwareResult() != null
-                    && cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND;
-        }
-
-        private boolean isUploadable(AbstractFile af) {
-            long size = af.getSize();
-            return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
-        }
-
-        private boolean uploadFile(SleuthkitCase skCase, DecryptedLicenseResponse decrypted, CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
-            if (!isUnknown(cloudBean)) {
-                return false;
-            }
-
-            AbstractFile af = skCase.getAbstractFileById(objId);
-            if (af == null) {
-                return false;
-            }
-
-            if (!isUploadable(af)) {
-                return false;
-            }
-
-            // get auth token / file upload url
-            AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
-            if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
-                throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
-            }
-
-            // upload bytes
-            ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
-            ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
-
-            // upload metadata
-            MetadataUploadRequest metaRequest = new MetadataUploadRequest()
-                    .setCreatedDate(af.getCrtime())
-                    .setFilePath(af.getUniquePath())
-                    .setFileSizeBytes(af.getSize())
-                    .setFileUploadUrl(authTokenResponse.getFileUploadUrl())
-                    .setMd5(af.getMd5Hash())
-                    .setSha1(af.getSha1Hash())
-                    .setSha256(af.getSha256Hash());
-
-            ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
-            return true;
-        }
-
         @Messages({
             "MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout",
             "MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out"
@@ -391,6 +348,82 @@ public class MalwareScanIngestModule implements FileIngestModule {
             return StringUtils.defaultString(orig).trim().toLowerCase();
         }
 
+        private boolean isUnknown(CTCloudBean cloudBean) {
+            return cloudBean != null
+                    && cloudBean.getMalwareResult() != null
+                    && cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND;
+        }
+
+        private boolean isUploadable(AbstractFile af) {
+            long size = af.getSize();
+            return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
+        }
+
+        private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
+            if (!uploadUnknownFiles) {
+                return false;
+            }
+
+            if (!isUnknown(cloudBean)) {
+                return false;
+            }
+
+            AbstractFile af = skCase.getAbstractFileById(objId);
+            if (af == null) {
+                return false;
+            }
+
+            if (!isUploadable(af)) {
+                return false;
+            }
+
+            // get auth token / file upload url
+            AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
+            if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
+                throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
+            } else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
+                // don't proceed with upload if reached limit
+                uploadUnknownFiles = false;
+                return false;
+            }
+
+            // upload bytes
+            ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
+            ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
+
+            // upload metadata
+            MetadataUploadRequest metaRequest = new MetadataUploadRequest()
+                    .setCreatedDate(af.getCrtime())
+                    .setFilePath(af.getUniquePath())
+                    .setFileSizeBytes(af.getSize())
+                    .setFileUploadUrl(authTokenResponse.getFileUploadUrl())
+                    .setMd5(af.getMd5Hash())
+                    .setSha1(af.getSha1Hash())
+                    .setSha256(af.getSha256Hash());
+
+            ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
+            return true;
+        }
+
+        private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) {
+            Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping);
+
+            for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) {
+                List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE);
+                for (List<String> batch : md5Batches) {
+                    // TODO query and capture still unknown
+                }
+                
+                if (remaining.isEmpty()) {
+                    return true;
+                }
+                
+               
+                Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
+            }
+
+        }
+
         @Messages({
             "MalwareScanIngestModule_SharedProcessing_createAnalysisResult_Yes=YES",
             "MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No=NO"