mirror of
https://github.com/overcuriousity/autopsy-flatpak.git
synced 2025-07-06 21:00:22 +00:00
updates
This commit is contained in:
parent
5d581f117f
commit
fe26a6e00f
@ -97,14 +97,10 @@ public class CTLicensePersistence {
|
|||||||
File settingsFile = getMalwareIngestFile();
|
File settingsFile = getMalwareIngestFile();
|
||||||
try {
|
try {
|
||||||
settingsFile.getParentFile().mkdirs();
|
settingsFile.getParentFile().mkdirs();
|
||||||
if (licenseResponse != null) {
|
objectMapper.writeValue(settingsFile, malwareIngestSettings);
|
||||||
objectMapper.writeValue(licenseFile, licenseResponse);
|
|
||||||
} else if (licenseFile.exists()) {
|
|
||||||
Files.delete(licenseFile.toPath());
|
|
||||||
}
|
|
||||||
return true;
|
return true;
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
logger.log(Level.WARNING, "There was an error writing CyberTriage license to file: " + licenseFile.getAbsolutePath(), ex);
|
logger.log(Level.WARNING, "There was an error writing malware ingest settings to file: " + settingsFile.getAbsolutePath(), ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -112,17 +108,21 @@ public class CTLicensePersistence {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public synchronized MalwareIngestSettings loadMalwareIngestSettings() {
|
public synchronized MalwareIngestSettings loadMalwareIngestSettings() {
|
||||||
Optional<LicenseResponse> toRet = Optional.empty();
|
MalwareIngestSettings settings = null;
|
||||||
File licenseFile = getCTLicenseFile();
|
File settingsFile = getMalwareIngestFile();
|
||||||
if (licenseFile.exists() && licenseFile.isFile()) {
|
if (settingsFile.exists() && settingsFile.isFile()) {
|
||||||
try {
|
try {
|
||||||
toRet = Optional.ofNullable(objectMapper.readValue(licenseFile, LicenseResponse.class));
|
settings = objectMapper.readValue(settingsFile, MalwareIngestSettings.class);
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
logger.log(Level.WARNING, "There was an error reading CyberTriage license to file: " + licenseFile.getAbsolutePath(), ex);
|
logger.log(Level.WARNING, "There was an error reading malware ingest settings from file: " + settingsFile.getAbsolutePath(), ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return toRet;
|
if (settings == null) {
|
||||||
|
settings = new MalwareIngestSettings();
|
||||||
|
}
|
||||||
|
|
||||||
|
return settings;
|
||||||
}
|
}
|
||||||
|
|
||||||
private File getCTLicenseFile() {
|
private File getCTLicenseFile() {
|
||||||
|
@ -111,6 +111,7 @@ public class CTMalwareScannerOptionsPanel extends CTOptionsSubPanel {
|
|||||||
@Override
|
@Override
|
||||||
public synchronized void saveSettings() {
|
public synchronized void saveSettings() {
|
||||||
ctPersistence.saveLicenseResponse(getLicenseInfo());
|
ctPersistence.saveLicenseResponse(getLicenseInfo());
|
||||||
|
ctPersistence.saveMalwareSettings(getIngestSettings());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -127,12 +128,27 @@ public class CTMalwareScannerOptionsPanel extends CTOptionsSubPanel {
|
|||||||
if (licenseInfo != null) {
|
if (licenseInfo != null) {
|
||||||
loadMalwareScansInfo(licenseInfo);
|
loadMalwareScansInfo(licenseInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
MalwareIngestSettings ingestSettings = ctPersistence.loadMalwareIngestSettings();
|
||||||
|
setIngestSettings(ingestSettings);
|
||||||
}
|
}
|
||||||
|
|
||||||
private synchronized LicenseResponse getLicenseInfo() {
|
private synchronized LicenseResponse getLicenseInfo() {
|
||||||
return this.licenseInfo == null ? null : this.licenseInfo.getLicenseResponse();
|
return this.licenseInfo == null ? null : this.licenseInfo.getLicenseResponse();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private MalwareIngestSettings getIngestSettings() {
|
||||||
|
return new MalwareIngestSettings()
|
||||||
|
.setUploadFiles(this.fileUploadCheckbox.isSelected());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void setIngestSettings(MalwareIngestSettings ingestSettings) {
|
||||||
|
if (ingestSettings == null) {
|
||||||
|
ingestSettings = new MalwareIngestSettings();
|
||||||
|
}
|
||||||
|
this.fileUploadCheckbox.setSelected(ingestSettings.isUploadFiles());
|
||||||
|
}
|
||||||
|
|
||||||
private synchronized void setLicenseDisplay(LicenseInfo licenseInfo, String licenseMessage) {
|
private synchronized void setLicenseDisplay(LicenseInfo licenseInfo, String licenseMessage) {
|
||||||
this.licenseInfo = licenseInfo;
|
this.licenseInfo = licenseInfo;
|
||||||
this.licenseInfoMessage = licenseMessage;
|
this.licenseInfoMessage = licenseMessage;
|
||||||
@ -411,7 +427,7 @@ public class CTMalwareScannerOptionsPanel extends CTOptionsSubPanel {
|
|||||||
}//GEN-LAST:event_licenseInfoAddButtonActionPerformed
|
}//GEN-LAST:event_licenseInfoAddButtonActionPerformed
|
||||||
|
|
||||||
private void fileUploadCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_fileUploadCheckboxActionPerformed
|
private void fileUploadCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_fileUploadCheckboxActionPerformed
|
||||||
// TODO add your handling code here:
|
this.firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
|
||||||
}//GEN-LAST:event_fileUploadCheckboxActionPerformed
|
}//GEN-LAST:event_fileUploadCheckboxActionPerformed
|
||||||
|
|
||||||
@NbBundle.Messages({
|
@NbBundle.Messages({
|
||||||
|
@ -40,6 +40,7 @@ import java.util.stream.Collectors;
|
|||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
import org.apache.commons.collections4.CollectionUtils;
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.curator.shaded.com.google.common.collect.Lists;
|
||||||
import org.openide.util.NbBundle.Messages;
|
import org.openide.util.NbBundle.Messages;
|
||||||
import org.sleuthkit.autopsy.casemodule.Case;
|
import org.sleuthkit.autopsy.casemodule.Case;
|
||||||
import org.sleuthkit.autopsy.coreutils.Logger;
|
import org.sleuthkit.autopsy.coreutils.Logger;
|
||||||
@ -96,6 +97,9 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
|||||||
|
|
||||||
private static final long MIN_UPLOAD_SIZE = 1;
|
private static final long MIN_UPLOAD_SIZE = 1;
|
||||||
private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
|
private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
|
||||||
|
private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5;
|
||||||
|
private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
|
||||||
|
|
||||||
|
|
||||||
private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
|
private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
|
||||||
"application/x-bat",//NON-NLS
|
"application/x-bat",//NON-NLS
|
||||||
@ -128,6 +132,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
|||||||
private BlackboardArtifact.Type malwareType = null;
|
private BlackboardArtifact.Type malwareType = null;
|
||||||
private long dsId = 0;
|
private long dsId = 0;
|
||||||
private long ingestJobId = 0;
|
private long ingestJobId = 0;
|
||||||
|
private boolean uploadUnknownFiles = false;
|
||||||
|
|
||||||
@Messages({
|
@Messages({
|
||||||
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
|
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
|
||||||
@ -186,6 +191,7 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
|||||||
dsId = context.getDataSource().getId();
|
dsId = context.getDataSource().getId();
|
||||||
ingestJobId = context.getJobId();
|
ingestJobId = context.getJobId();
|
||||||
licenseInfo = licenseInfoOpt.get();
|
licenseInfo = licenseInfoOpt.get();
|
||||||
|
uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
|
||||||
|
|
||||||
// set run state to initialized
|
// set run state to initialized
|
||||||
runState = RunState.STARTED_UP;
|
runState = RunState.STARTED_UP;
|
||||||
@ -201,55 +207,6 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
|||||||
return limit - used;
|
return limit - used;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isUnknown(CTCloudBean cloudBean) {
|
|
||||||
return cloudBean != null
|
|
||||||
&& cloudBean.getMalwareResult() != null
|
|
||||||
&& cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isUploadable(AbstractFile af) {
|
|
||||||
long size = af.getSize();
|
|
||||||
return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean uploadFile(SleuthkitCase skCase, DecryptedLicenseResponse decrypted, CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
|
|
||||||
if (!isUnknown(cloudBean)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
AbstractFile af = skCase.getAbstractFileById(objId);
|
|
||||||
if (af == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isUploadable(af)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// get auth token / file upload url
|
|
||||||
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
|
|
||||||
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
|
|
||||||
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
|
|
||||||
}
|
|
||||||
|
|
||||||
// upload bytes
|
|
||||||
ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
|
|
||||||
ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
|
|
||||||
|
|
||||||
// upload metadata
|
|
||||||
MetadataUploadRequest metaRequest = new MetadataUploadRequest()
|
|
||||||
.setCreatedDate(af.getCrtime())
|
|
||||||
.setFilePath(af.getUniquePath())
|
|
||||||
.setFileSizeBytes(af.getSize())
|
|
||||||
.setFileUploadUrl(authTokenResponse.getFileUploadUrl())
|
|
||||||
.setMd5(af.getMd5Hash())
|
|
||||||
.setSha1(af.getSha1Hash())
|
|
||||||
.setSha256(af.getSha256Hash());
|
|
||||||
|
|
||||||
ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Messages({
|
@Messages({
|
||||||
"MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout",
|
"MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout",
|
||||||
"MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out"
|
"MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out"
|
||||||
@ -391,6 +348,82 @@ public class MalwareScanIngestModule implements FileIngestModule {
|
|||||||
return StringUtils.defaultString(orig).trim().toLowerCase();
|
return StringUtils.defaultString(orig).trim().toLowerCase();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private boolean isUnknown(CTCloudBean cloudBean) {
|
||||||
|
return cloudBean != null
|
||||||
|
&& cloudBean.getMalwareResult() != null
|
||||||
|
&& cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isUploadable(AbstractFile af) {
|
||||||
|
long size = af.getSize();
|
||||||
|
return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
|
||||||
|
if (!uploadUnknownFiles) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isUnknown(cloudBean)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
AbstractFile af = skCase.getAbstractFileById(objId);
|
||||||
|
if (af == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isUploadable(af)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// get auth token / file upload url
|
||||||
|
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
|
||||||
|
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
|
||||||
|
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
|
||||||
|
} else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
|
||||||
|
// don't proceed with upload if reached limit
|
||||||
|
uploadUnknownFiles = false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// upload bytes
|
||||||
|
ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
|
||||||
|
ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
|
||||||
|
|
||||||
|
// upload metadata
|
||||||
|
MetadataUploadRequest metaRequest = new MetadataUploadRequest()
|
||||||
|
.setCreatedDate(af.getCrtime())
|
||||||
|
.setFilePath(af.getUniquePath())
|
||||||
|
.setFileSizeBytes(af.getSize())
|
||||||
|
.setFileUploadUrl(authTokenResponse.getFileUploadUrl())
|
||||||
|
.setMd5(af.getMd5Hash())
|
||||||
|
.setSha1(af.getSha1Hash())
|
||||||
|
.setSha256(af.getSha256Hash());
|
||||||
|
|
||||||
|
ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) {
|
||||||
|
Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping);
|
||||||
|
|
||||||
|
for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) {
|
||||||
|
List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE);
|
||||||
|
for (List<String> batch : md5Batches) {
|
||||||
|
// TODO query and capture still unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
if (remaining.isEmpty()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
@Messages({
|
@Messages({
|
||||||
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_Yes=YES",
|
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_Yes=YES",
|
||||||
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No=NO"
|
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No=NO"
|
||||||
|
Loading…
x
Reference in New Issue
Block a user