mirror of
https://github.com/overcuriousity/autopsy-flatpak.git
synced 2025-07-06 21:00:22 +00:00
updates for sha1 hashing
This commit is contained in:
Greg DiCristofaro
parent
8917867778
commit
3e2bbc6421
@ -200,6 +200,7 @@ public class CTCloudHttpClient {
|
||||
public void doFileUploadPost(String urlPath, String fileName, InputStream fileIs) throws CTCloudException {
|
||||
|
||||
try (CloseableHttpClient httpclient = createConnection(getProxySettings(), sslContext)) {
|
||||
LOGGER.log(Level.INFO, "initiating http post request to ctcloud server " + urlPath);
|
||||
HttpPost post = new HttpPost(urlPath);
|
||||
configureRequestTimeout(post);
|
||||
|
||||
|
||||
@ -27,10 +27,14 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
|
||||
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
|
||||
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
|
||||
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
|
||||
import java.security.DigestInputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.text.MessageFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
@ -133,6 +137,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
|
||||
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
|
||||
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
|
||||
private final UsernameAnonymizer usernameAnonymizer = new UsernameAnonymizer();
|
||||
|
||||
private IngestJobState ingestJobState = null;
|
||||
|
||||
@ -258,31 +263,86 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
* @param af The abstract file.
|
||||
* @return The md5 hash (or null if could not be determined).
|
||||
*/
|
||||
private static String getOrCalcHash(AbstractFile af) {
|
||||
private static String getOrCalcHash(AbstractFile af, HashType hashType) {
|
||||
switch (hashType) {
|
||||
case MD5:
|
||||
if (StringUtils.isNotBlank(af.getMd5Hash())) {
|
||||
return af.getMd5Hash();
|
||||
}
|
||||
break;
|
||||
case SHA256:
|
||||
if (StringUtils.isNotBlank(af.getSha256Hash())) {
|
||||
return af.getSha256Hash();
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
List<HashResult> hashResults = HashUtility.calculateHashes(af, Collections.singletonList(HashType.MD5));
|
||||
List<HashResult> hashResults = HashUtility.calculateHashes(af, Collections.singletonList(hashType));
|
||||
if (CollectionUtils.isNotEmpty(hashResults)) {
|
||||
for (HashResult hashResult : hashResults) {
|
||||
if (hashResult.getType() == HashType.MD5) {
|
||||
if (hashResult.getType() == hashType) {
|
||||
return hashResult.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (TskCoreException ex) {
|
||||
logger.log(Level.WARNING,
|
||||
MessageFormat.format("An error occurred while processing file name: {0} and obj id: {1}.",
|
||||
MessageFormat.format("An error occurred while processing hash for file name: {0} and obj id: {1} and hash type {2}.",
|
||||
af.getName(),
|
||||
af.getId()),
|
||||
af.getId(),
|
||||
hashType.name()),
|
||||
ex);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets or calculates the md5 for a file.
|
||||
*
|
||||
* @param af The file.
|
||||
* @return The hash.
|
||||
*/
|
||||
private static String getOrCalcMd5(AbstractFile af) {
|
||||
return getOrCalcHash(af, HashType.MD5);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets or calculates the sha256 for a file.
|
||||
*
|
||||
* @param af The file.
|
||||
* @return The hash.
|
||||
*/
|
||||
private static String getOrCalcSha256(AbstractFile af) {
|
||||
return getOrCalcHash(af, HashType.SHA256);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets or calculates the sha1 for a file.
|
||||
*
|
||||
* @param af The file.
|
||||
* @return The hash.
|
||||
*/
|
||||
private static String getOrCalcSha1(AbstractFile af) throws NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
|
||||
if (StringUtils.isNotBlank(af.getSha1Hash())) {
|
||||
return af.getSha1Hash();
|
||||
}
|
||||
// taken from https://stackoverflow.com/questions/6293713/java-how-to-create-sha-1-for-a-file
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-1");
|
||||
ReadContentInputStream afStream = new ReadContentInputStream(af);
|
||||
int n = 0;
|
||||
byte[] buffer = new byte[8192];
|
||||
while (n != -1) {
|
||||
n = afStream.read(buffer);
|
||||
if (n > 0) {
|
||||
digest.update(buffer, 0, n);
|
||||
}
|
||||
}
|
||||
byte[] hashBytes = digest.digest();
|
||||
String hashString = HexFormat.of().formatHex(hashBytes);
|
||||
return hashString;
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes a file. The file goes through the lookup process if the
|
||||
* file meets acceptable criteria: 1) not FileKnown.KNOWN 2) is
|
||||
@ -305,7 +365,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
&& EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(ingestJobState.getFileTypeDetector().getMIMEType(af)).trim().toLowerCase())
|
||||
&& CollectionUtils.isEmpty(af.getAnalysisResults(ingestJobState.getMalwareType()))) {
|
||||
|
||||
String md5 = getOrCalcHash(af);
|
||||
String md5 = getOrCalcMd5(af);
|
||||
if (StringUtils.isNotBlank(md5)) {
|
||||
batchProcessor.add(new FileRecord(af.getId(), md5));
|
||||
}
|
||||
@ -393,12 +453,11 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
* @param repResult The ct cloud results.
|
||||
* @throws org.sleuthkit.datamodel.Blackboard.BlackboardException
|
||||
* @throws TskCoreException
|
||||
* @throws TskCoreException
|
||||
*/
|
||||
@Messages({
|
||||
"MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_title=Some Lookup Results Not Processed",
|
||||
"MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_desc=Some lookup results were not processed due to exceeding limits. Please try again later.",})
|
||||
private void handleLookupResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> repResult) throws Blackboard.BlackboardException, TskCoreException, TskCoreException, CTCloudException {
|
||||
private void handleLookupResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> repResult) throws Blackboard.BlackboardException, TskCoreException, TskCoreException, CTCloudException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
|
||||
if (CollectionUtils.isEmpty(repResult)) {
|
||||
return;
|
||||
}
|
||||
@ -445,7 +504,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
* @param performFileUpload True if the class of results warrants file
|
||||
* upload (i.e. NOT_FOUND)
|
||||
*/
|
||||
private void handleNonFoundResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> results, boolean performFileUpload) throws CTCloudException, TskCoreException {
|
||||
private void handleNonFoundResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> results, boolean performFileUpload) throws CTCloudException, TskCoreException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
|
||||
if (CollectionUtils.isNotEmpty(results)
|
||||
&& ingestJobState.isDoFileLookups()
|
||||
&& ((performFileUpload && ingestJobState.isUploadUnknownFiles()) || (!performFileUpload && ingestJobState.isQueryForMissing()))) {
|
||||
@ -539,7 +598,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
* @throws CTCloudException
|
||||
* @throws TskCoreException
|
||||
*/
|
||||
private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId) throws CTCloudException, TskCoreException {
|
||||
private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId) throws CTCloudException, TskCoreException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
|
||||
if (!ingestJobState.isUploadUnknownFiles() || ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
|
||||
return false;
|
||||
}
|
||||
@ -568,13 +627,12 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
// upload metadata
|
||||
MetadataUploadRequest metaRequest = new MetadataUploadRequest()
|
||||
.setCreatedDate(af.getCrtime() == 0 ? null : af.getCrtime())
|
||||
.setFilePath(af.getUniquePath())
|
||||
.setFilePath(usernameAnonymizer.anonymousUsername(af.getUniquePath()))
|
||||
.setFileSizeBytes(af.getSize())
|
||||
.setFileUploadUrl(authTokenResponse.getFileUploadUrl())
|
||||
.setMd5(md5)
|
||||
// these may be missing, but that's fine
|
||||
.setSha1(af.getSha1Hash())
|
||||
.setSha256(af.getSha256Hash());
|
||||
.setSha1(getOrCalcSha1(af))
|
||||
.setSha256(getOrCalcSha256(af));
|
||||
|
||||
ctApiDAO.uploadMeta(new AuthenticatedRequestData(ingestJobState.getLicenseInfo().getDecryptedLicense(), authTokenResponse), metaRequest);
|
||||
return true;
|
||||
@ -627,7 +685,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
createAnalysisResults(ingestJobState, found, remaining);
|
||||
|
||||
// remove any found items from the list of items to long poll for
|
||||
for (CTCloudBean foundItem : found) {
|
||||
for (CTCloudBean foundItem : CollectionUtils.emptyIfNull(found)) {
|
||||
String normalizedMd5 = normalizedMd5(foundItem.getMd5HashValue());
|
||||
remaining.remove(normalizedMd5);
|
||||
}
|
||||
@ -768,8 +826,8 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
||||
|
||||
// flush any remaining items
|
||||
try {
|
||||
longPollForNotFound(ingestJobState);
|
||||
batchProcessor.flushAndReset();
|
||||
longPollForNotFound(ingestJobState);
|
||||
} catch (InterruptedException ex) {
|
||||
notifyWarning(
|
||||
Bundle.MalwareScanIngestModule_SharedProcessing_flushTimeout_title(),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user