From b3bafd8a225916d2110fccf153d2161fa590fbfa Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 21 Aug 2023 11:59:47 -0400 Subject: [PATCH 1/2] file upload api changes --- .../cybertriage/autopsy/ctapi/CTApiDAO.java | 9 ++--- .../autopsy/ctapi/CTCloudHttpClient.java | 33 +++++++++++-------- .../autopsy/ctapi/json/AuthTokenRequest.java | 13 ++++++++ .../malwarescan/MalwareScanIngestModule.java | 6 ++-- 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java index c358feec61..e17e771b2a 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java @@ -78,13 +78,14 @@ public class CTApiDAO { } public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted) throws CTCloudException { - return getAuthToken(decrypted, false); + return getAuthToken(decrypted, null); } - public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, boolean fileUpload) throws CTCloudException { + public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, Long fileUploadSize) throws CTCloudException { AuthTokenRequest authTokenRequest = new AuthTokenRequest() .setAutopsyVersion(getAppVersion()) - .setRequestFileUpload(fileUpload) + .setRequestFileUpload(fileUploadSize != null && fileUploadSize > 0) + .setFileUploadSize(fileUploadSize != null && fileUploadSize > 0 ? fileUploadSize : null) .setBoostLicenseId(decrypted.getBoostLicenseId()) .setHostId(decrypted.getLicenseHostId()); @@ -92,7 +93,7 @@ public class CTApiDAO { } public void uploadFile(String url, String fileName, InputStream fileIs) throws CTCloudException { - httpClient.doFileUploadPost(url, fileName, fileIs); + httpClient.doFileUploadPut(url, fileName, fileIs); } public void uploadMeta(AuthenticatedRequestData authenticatedRequestData, MetadataUploadRequest metaRequest) throws CTCloudException { diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java index 0a280c87e8..a5a1e0b554 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java @@ -55,11 +55,13 @@ import org.apache.http.HttpStatus; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.sleuthkit.autopsy.coreutils.Logger; @@ -184,7 +186,7 @@ class CTCloudHttpClient { return null; } - public void doFileUploadPost(String fullUrlPath, String fileName, InputStream fileIs) throws CTCloudException { + public void doFileUploadPut(String fullUrlPath, String fileName, InputStream fileIs) throws CTCloudException { URI postUri; try { postUri = new URI(fullUrlPath); @@ -195,23 +197,26 @@ class CTCloudHttpClient { try (CloseableHttpClient httpclient = createConnection(proxySelector, sslContext)) { LOGGER.log(Level.INFO, "initiating http post request to ctcloud server " + fullUrlPath); - HttpPost post = new HttpPost(postUri); - configureRequestTimeout(post); + HttpPut put = new HttpPut(postUri); + configureRequestTimeout(put); - post.addHeader("Connection", "keep-alive"); + put.addHeader("Connection", "keep-alive"); + put.addHeader("Content-Type", "application/octet-stream"); - MultipartEntityBuilder builder = MultipartEntityBuilder.create(); - builder.addBinaryBody( - "file", - fileIs, - ContentType.APPLICATION_OCTET_STREAM, - fileName - ); +// MultipartEntityBuilder builder = MultipartEntityBuilder.create(); +// builder.addBinaryBody( +// "file", +// fileBytes, +// ContentType.APPLICATION_OCTET_STREAM, +// file.getFileName() +// ); +// +// HttpEntity multipart = builder.build(); +// post.setEntity(multipart); - HttpEntity multipart = builder.build(); - post.setEntity(multipart); + put.setEntity(new InputStreamEntity(fileIs)); - try (CloseableHttpResponse response = httpclient.execute(post)) { + try (CloseableHttpResponse response = httpclient.execute(put)) { int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NO_CONTENT) { LOGGER.log(Level.INFO, "Response Received. - Status OK"); diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java index 6a093fee8e..f137818346 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java @@ -34,6 +34,9 @@ public class AuthTokenRequest { @JsonProperty("requestFileUpload") private boolean requestFileUpload; + @JsonProperty("fileUploadSize") + private Long fileUploadSize; + @JsonProperty("host_id") private String hostId; @@ -64,6 +67,16 @@ public class AuthTokenRequest { return this; } + public Long getFileUploadSize() { + return fileUploadSize; + } + + public AuthTokenRequest setFileUploadSize(Long fileUploadSize) { + this.fileUploadSize = fileUploadSize; + return this; + } + + public String getHostId() { return hostId; } diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java index 4698392478..cbe76ba235 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java @@ -105,8 +105,10 @@ class MalwareScanIngestModule implements FileIngestModule { //minimum file uploads left before issuing warning private static final long LOW_UPLOADS_REMAINING = 25; + // min and max upload size in bytes private static final long MIN_UPLOAD_SIZE = 1; - private static final long MAX_UPLOAD_SIZE = 1_000_000_000; + private static final long MAX_UPLOAD_SIZE = 100_000_000; // 100MB + private static final int NUM_FILE_UPLOAD_RETRIES = 7; private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000; @@ -634,7 +636,7 @@ class MalwareScanIngestModule implements FileIngestModule { } // get auth token / file upload url - AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), true); + AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), af.getSize()); if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) { throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR); } else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) { From 14d8a09288eeadb7d20afd6afa7672e52bdebe24 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 21 Aug 2023 16:35:24 -0400 Subject: [PATCH 2/2] file upload changes --- .../cybertriage/autopsy/ctapi/CTApiDAO.java | 5 +- .../autopsy/ctapi/CTCloudHttpClient.java | 41 +++++------ .../autopsy/ctapi/json/FileUploadRequest.java | 69 +++++++++++++++++++ .../malwarescan/MalwareScanIngestModule.java | 13 +++- 4 files changed, 103 insertions(+), 25 deletions(-) create mode 100644 Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java index e17e771b2a..f21873f565 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java @@ -25,6 +25,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean; import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBeanResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.FileReputationRequest; +import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest; import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseRequest; import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest; @@ -92,8 +93,8 @@ public class CTApiDAO { return httpClient.doPost(AUTH_TOKEN_REQUEST_PATH, authTokenRequest, AuthTokenResponse.class); } - public void uploadFile(String url, String fileName, InputStream fileIs) throws CTCloudException { - httpClient.doFileUploadPut(url, fileName, fileIs); + public void uploadFile(FileUploadRequest fileUploadRequest) throws CTCloudException { + httpClient.doFileUploadPut(fileUploadRequest); } public void uploadMeta(AuthenticatedRequestData authenticatedRequestData, MetadataUploadRequest metaRequest) throws CTCloudException { diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java index a5a1e0b554..0a7c69cdee 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java @@ -18,6 +18,8 @@ */ package com.basistech.df.cybertriage.autopsy.ctapi; +import com.basistech.df.cybertriage.autopsy.ctapi.CTCloudException.ErrorCode; +import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest; import com.basistech.df.cybertriage.autopsy.ctapi.util.ObjectMapperUtil; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -63,7 +65,6 @@ import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.ContentType; import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; -import org.apache.http.entity.mime.MultipartEntityBuilder; import org.sleuthkit.autopsy.coreutils.Logger; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; @@ -186,10 +187,23 @@ class CTCloudHttpClient { return null; } - public void doFileUploadPut(String fullUrlPath, String fileName, InputStream fileIs) throws CTCloudException { - URI postUri; + public void doFileUploadPut(FileUploadRequest fileUploadRequest) throws CTCloudException { + if (fileUploadRequest == null) { + throw new CTCloudException(ErrorCode.BAD_REQUEST, new IllegalArgumentException("fileUploadRequest cannot be null")); + } + + String fullUrlPath = fileUploadRequest.getFullUrlPath(); + String fileName = fileUploadRequest.getFileName(); + InputStream fileInputStream = fileUploadRequest.getFileInputStream(); + Long contentLength = fileUploadRequest.getContentLength(); + + if (StringUtils.isBlank(fullUrlPath) || fileInputStream == null || contentLength == null || contentLength <= 0) { + throw new CTCloudException(ErrorCode.BAD_REQUEST, new IllegalArgumentException("fullUrlPath, fileInputStream, contentLength must not be empty, null or less than 0")); + } + + URI putUri; try { - postUri = new URI(fullUrlPath); + putUri = new URI(fullUrlPath); } catch (URISyntaxException ex) { LOGGER.log(Level.WARNING, "Wrong URL syntax for CT Cloud " + fullUrlPath, ex); throw new CTCloudException(CTCloudException.ErrorCode.UNKNOWN, ex); @@ -197,24 +211,11 @@ class CTCloudHttpClient { try (CloseableHttpClient httpclient = createConnection(proxySelector, sslContext)) { LOGGER.log(Level.INFO, "initiating http post request to ctcloud server " + fullUrlPath); - HttpPut put = new HttpPut(postUri); + HttpPut put = new HttpPut(putUri); configureRequestTimeout(put); put.addHeader("Connection", "keep-alive"); - put.addHeader("Content-Type", "application/octet-stream"); - -// MultipartEntityBuilder builder = MultipartEntityBuilder.create(); -// builder.addBinaryBody( -// "file", -// fileBytes, -// ContentType.APPLICATION_OCTET_STREAM, -// file.getFileName() -// ); -// -// HttpEntity multipart = builder.build(); -// post.setEntity(multipart); - - put.setEntity(new InputStreamEntity(fileIs)); + put.setEntity(new InputStreamEntity(fileInputStream, contentLength, ContentType.APPLICATION_OCTET_STREAM)); try (CloseableHttpResponse response = httpclient.execute(put)) { int statusCode = response.getStatusLine().getStatusCode(); @@ -386,7 +387,7 @@ class CTCloudHttpClient { HttpClientBuilder builder; if (ProxySettings.getProxyType() != ProxySettings.DIRECT_CONNECTION - && StringUtils.isBlank(ProxySettings.getAuthenticationUsername()) + && StringUtils.isBlank(ProxySettings.getAuthenticationUsername()) && ArrayUtils.isEmpty(ProxySettings.getAuthenticationPassword()) && WinHttpClients.isWinAuthAvailable()) { diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java new file mode 100644 index 0000000000..48d8dc77b4 --- /dev/null +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java @@ -0,0 +1,69 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2023 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.basistech.df.cybertriage.autopsy.ctapi.json; + +import java.io.InputStream; + +/** + * Data for a file upload request. + */ +public class FileUploadRequest { + + private String fullUrlPath; + private String fileName; + private InputStream fileInputStream; + private Long contentLength; + + public String getFullUrlPath() { + return fullUrlPath; + } + + public FileUploadRequest setFullUrlPath(String fullUrlPath) { + this.fullUrlPath = fullUrlPath; + return this; + } + + public String getFileName() { + return fileName; + } + + public FileUploadRequest setFileName(String fileName) { + this.fileName = fileName; + return this; + } + + public InputStream getFileInputStream() { + return fileInputStream; + } + + public FileUploadRequest setFileInputStream(InputStream fileInputStream) { + this.fileInputStream = fileInputStream; + return this; + } + + public Long getContentLength() { + return contentLength; + } + + public FileUploadRequest setContentLength(Long contentLength) { + this.contentLength = contentLength; + return this; + } + +} diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java index cbe76ba235..c66b88cce9 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java @@ -23,6 +23,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.CTCloudException; import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthTokenResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthenticatedRequestData; import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean; +import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest; import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo; import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status; import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest; @@ -108,7 +109,7 @@ class MalwareScanIngestModule implements FileIngestModule { // min and max upload size in bytes private static final long MIN_UPLOAD_SIZE = 1; private static final long MAX_UPLOAD_SIZE = 100_000_000; // 100MB - + private static final int NUM_FILE_UPLOAD_RETRIES = 7; private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000; @@ -654,7 +655,13 @@ class MalwareScanIngestModule implements FileIngestModule { // upload bytes ReadContentInputStream fileInputStream = new ReadContentInputStream(af); - ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream); + + ctApiDAO.uploadFile(new FileUploadRequest() + .setContentLength(af.getSize()) + .setFileInputStream(fileInputStream) + .setFileName(af.getName()) + .setFullUrlPath(authTokenResponse.getFileUploadUrl()) + ); // upload metadata MetadataUploadRequest metaRequest = new MetadataUploadRequest() @@ -843,7 +850,7 @@ class MalwareScanIngestModule implements FileIngestModule { : Bundle.MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No(); String justification = cloudBean.getMalwareResult().getStatusDescription(); - + return ingestJobState.getTskCase().getBlackboard().newAnalysisResult( ingestJobState.getMalwareType(), objId,