updates for sha1 hashing

Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java

@@ -200,6 +200,7 @@ public class CTCloudHttpClient {
     public void doFileUploadPost(String urlPath, String fileName, InputStream fileIs) throws CTCloudException {
          
         try (CloseableHttpClient httpclient = createConnection(getProxySettings(), sslContext)) {
+            LOGGER.log(Level.INFO, "initiating http post request to ctcloud server " + urlPath);
             HttpPost post = new HttpPost(urlPath);
             configureRequestTimeout(post);
             

Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java

@@ -27,10 +27,14 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
 import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HexFormat;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -133,6 +137,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
 
         private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
         private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
+        private final UsernameAnonymizer usernameAnonymizer = new UsernameAnonymizer();
 
         private IngestJobState ingestJobState = null;
 
@@ -258,31 +263,86 @@ public class MalwareScanIngestModule implements FileIngestModule {
          * @param af The abstract file.
          * @return The md5 hash (or null if could not be determined).
          */
-        private static String getOrCalcHash(AbstractFile af) {
+        private static String getOrCalcHash(AbstractFile af, HashType hashType) {
-            if (StringUtils.isNotBlank(af.getMd5Hash())) {
+            switch (hashType) {
-                return af.getMd5Hash();
+                case MD5:
+                    if (StringUtils.isNotBlank(af.getMd5Hash())) {
+                        return af.getMd5Hash();
+                    }
+                    break;
+                case SHA256:
+                    if (StringUtils.isNotBlank(af.getSha256Hash())) {
+                        return af.getSha256Hash();
+                    }
             }
 
             try {
-                List<HashResult> hashResults = HashUtility.calculateHashes(af, Collections.singletonList(HashType.MD5));
+                List<HashResult> hashResults = HashUtility.calculateHashes(af, Collections.singletonList(hashType));
                 if (CollectionUtils.isNotEmpty(hashResults)) {
                     for (HashResult hashResult : hashResults) {
-                        if (hashResult.getType() == HashType.MD5) {
+                        if (hashResult.getType() == hashType) {
                             return hashResult.getValue();
                         }
                     }
                 }
             } catch (TskCoreException ex) {
                 logger.log(Level.WARNING,
-                        MessageFormat.format("An error occurred while processing file name: {0} and obj id: {1}.",
+                        MessageFormat.format("An error occurred while processing hash for file name: {0} and obj id: {1} and hash type {2}.",
                                 af.getName(),
-                                af.getId()),
+                                af.getId(),
+                                hashType.name()),
                         ex);
             }
 
             return null;
         }
 
+        /**
+         * Gets or calculates the md5 for a file.
+         *
+         * @param af The file.
+         * @return The hash.
+         */
+        private static String getOrCalcMd5(AbstractFile af) {
+            return getOrCalcHash(af, HashType.MD5);
+        }
+
+        /**
+         * Gets or calculates the sha256 for a file.
+         *
+         * @param af The file.
+         * @return The hash.
+         */
+        private static String getOrCalcSha256(AbstractFile af) {
+            return getOrCalcHash(af, HashType.SHA256);
+        }
+
+        /**
+         * Gets or calculates the sha1 for a file.
+         *
+         * @param af The file.
+         * @return The hash.
+         */
+        private static String getOrCalcSha1(AbstractFile af) throws NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
+            if (StringUtils.isNotBlank(af.getSha1Hash())) {
+                return af.getSha1Hash();
+            }
+            // taken from https://stackoverflow.com/questions/6293713/java-how-to-create-sha-1-for-a-file
+            MessageDigest digest = MessageDigest.getInstance("SHA-1");
+            ReadContentInputStream afStream = new ReadContentInputStream(af);
+            int n = 0;
+            byte[] buffer = new byte[8192];
+            while (n != -1) {
+                n = afStream.read(buffer);
+                if (n > 0) {
+                    digest.update(buffer, 0, n);
+                }
+            }
+            byte[] hashBytes = digest.digest();
+            String hashString = HexFormat.of().formatHex(hashBytes);
+            return hashString;
+        }
+
         /**
          * Processes a file. The file goes through the lookup process if the
          * file meets acceptable criteria: 1) not FileKnown.KNOWN 2) is
@@ -305,7 +365,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
                         && EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(ingestJobState.getFileTypeDetector().getMIMEType(af)).trim().toLowerCase())
                         && CollectionUtils.isEmpty(af.getAnalysisResults(ingestJobState.getMalwareType()))) {
 
-                    String md5 = getOrCalcHash(af);
+                    String md5 = getOrCalcMd5(af);
                     if (StringUtils.isNotBlank(md5)) {
                         batchProcessor.add(new FileRecord(af.getId(), md5));
                     }
@@ -393,12 +453,11 @@ public class MalwareScanIngestModule implements FileIngestModule {
          * @param repResult The ct cloud results.
          * @throws org.sleuthkit.datamodel.Blackboard.BlackboardException
          * @throws TskCoreException
-         * @throws TskCoreException
          */
         @Messages({
             "MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_title=Some Lookup Results Not Processed",
             "MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_desc=Some lookup results were not processed due to exceeding limits.  Please try again later.",})
-        private void handleLookupResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> repResult) throws Blackboard.BlackboardException, TskCoreException, TskCoreException, CTCloudException {
+        private void handleLookupResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> repResult) throws Blackboard.BlackboardException, TskCoreException, TskCoreException, CTCloudException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
             if (CollectionUtils.isEmpty(repResult)) {
                 return;
             }
@@ -445,7 +504,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
          * @param performFileUpload True if the class of results warrants file
          * upload (i.e. NOT_FOUND)
          */
-        private void handleNonFoundResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> results, boolean performFileUpload) throws CTCloudException, TskCoreException {
+        private void handleNonFoundResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> results, boolean performFileUpload) throws CTCloudException, TskCoreException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
             if (CollectionUtils.isNotEmpty(results)
                     && ingestJobState.isDoFileLookups()
                     && ((performFileUpload && ingestJobState.isUploadUnknownFiles()) || (!performFileUpload && ingestJobState.isQueryForMissing()))) {
@@ -539,7 +598,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
          * @throws CTCloudException
          * @throws TskCoreException
          */
-        private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId) throws CTCloudException, TskCoreException {
+        private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId) throws CTCloudException, TskCoreException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
             if (!ingestJobState.isUploadUnknownFiles() || ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
                 return false;
             }
@@ -568,13 +627,12 @@ public class MalwareScanIngestModule implements FileIngestModule {
             // upload metadata
             MetadataUploadRequest metaRequest = new MetadataUploadRequest()
                     .setCreatedDate(af.getCrtime() == 0 ? null : af.getCrtime())
-                    .setFilePath(af.getUniquePath())
+                    .setFilePath(usernameAnonymizer.anonymousUsername(af.getUniquePath()))
                     .setFileSizeBytes(af.getSize())
                     .setFileUploadUrl(authTokenResponse.getFileUploadUrl())
                     .setMd5(md5)
-                    // these may be missing, but that's fine
+                    .setSha1(getOrCalcSha1(af))
-                    .setSha1(af.getSha1Hash())
+                    .setSha256(getOrCalcSha256(af));
-                    .setSha256(af.getSha256Hash());
 
             ctApiDAO.uploadMeta(new AuthenticatedRequestData(ingestJobState.getLicenseInfo().getDecryptedLicense(), authTokenResponse), metaRequest);
             return true;
@@ -627,7 +685,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
                     createAnalysisResults(ingestJobState, found, remaining);
 
                     // remove any found items from the list of items to long poll for
-                    for (CTCloudBean foundItem : found) {
+                    for (CTCloudBean foundItem : CollectionUtils.emptyIfNull(found)) {
                         String normalizedMd5 = normalizedMd5(foundItem.getMd5HashValue());
                         remaining.remove(normalizedMd5);
                     }
@@ -768,8 +826,8 @@ public class MalwareScanIngestModule implements FileIngestModule {
 
             // flush any remaining items
             try {
-                longPollForNotFound(ingestJobState);
                 batchProcessor.flushAndReset();
+                longPollForNotFound(ingestJobState);
             } catch (InterruptedException ex) {
                 notifyWarning(
                         Bundle.MalwareScanIngestModule_SharedProcessing_flushTimeout_title(),