diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java index c358feec61..f21873f565 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java @@ -25,6 +25,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean; import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBeanResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.FileReputationRequest; +import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest; import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseRequest; import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest; @@ -78,21 +79,22 @@ public class CTApiDAO { } public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted) throws CTCloudException { - return getAuthToken(decrypted, false); + return getAuthToken(decrypted, null); } - public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, boolean fileUpload) throws CTCloudException { + public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, Long fileUploadSize) throws CTCloudException { AuthTokenRequest authTokenRequest = new AuthTokenRequest() .setAutopsyVersion(getAppVersion()) - .setRequestFileUpload(fileUpload) + .setRequestFileUpload(fileUploadSize != null && fileUploadSize > 0) + .setFileUploadSize(fileUploadSize != null && fileUploadSize > 0 ? fileUploadSize : null) .setBoostLicenseId(decrypted.getBoostLicenseId()) .setHostId(decrypted.getLicenseHostId()); return httpClient.doPost(AUTH_TOKEN_REQUEST_PATH, authTokenRequest, AuthTokenResponse.class); } - public void uploadFile(String url, String fileName, InputStream fileIs) throws CTCloudException { - httpClient.doFileUploadPost(url, fileName, fileIs); + public void uploadFile(FileUploadRequest fileUploadRequest) throws CTCloudException { + httpClient.doFileUploadPut(fileUploadRequest); } public void uploadMeta(AuthenticatedRequestData authenticatedRequestData, MetadataUploadRequest metaRequest) throws CTCloudException { diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java index 0a280c87e8..0a7c69cdee 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java @@ -18,6 +18,8 @@ */ package com.basistech.df.cybertriage.autopsy.ctapi; +import com.basistech.df.cybertriage.autopsy.ctapi.CTCloudException.ErrorCode; +import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest; import com.basistech.df.cybertriage.autopsy.ctapi.util.ObjectMapperUtil; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -55,13 +57,14 @@ import org.apache.http.HttpStatus; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; -import org.apache.http.entity.mime.MultipartEntityBuilder; import org.sleuthkit.autopsy.coreutils.Logger; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; @@ -184,10 +187,23 @@ class CTCloudHttpClient { return null; } - public void doFileUploadPost(String fullUrlPath, String fileName, InputStream fileIs) throws CTCloudException { - URI postUri; + public void doFileUploadPut(FileUploadRequest fileUploadRequest) throws CTCloudException { + if (fileUploadRequest == null) { + throw new CTCloudException(ErrorCode.BAD_REQUEST, new IllegalArgumentException("fileUploadRequest cannot be null")); + } + + String fullUrlPath = fileUploadRequest.getFullUrlPath(); + String fileName = fileUploadRequest.getFileName(); + InputStream fileInputStream = fileUploadRequest.getFileInputStream(); + Long contentLength = fileUploadRequest.getContentLength(); + + if (StringUtils.isBlank(fullUrlPath) || fileInputStream == null || contentLength == null || contentLength <= 0) { + throw new CTCloudException(ErrorCode.BAD_REQUEST, new IllegalArgumentException("fullUrlPath, fileInputStream, contentLength must not be empty, null or less than 0")); + } + + URI putUri; try { - postUri = new URI(fullUrlPath); + putUri = new URI(fullUrlPath); } catch (URISyntaxException ex) { LOGGER.log(Level.WARNING, "Wrong URL syntax for CT Cloud " + fullUrlPath, ex); throw new CTCloudException(CTCloudException.ErrorCode.UNKNOWN, ex); @@ -195,23 +211,13 @@ class CTCloudHttpClient { try (CloseableHttpClient httpclient = createConnection(proxySelector, sslContext)) { LOGGER.log(Level.INFO, "initiating http post request to ctcloud server " + fullUrlPath); - HttpPost post = new HttpPost(postUri); - configureRequestTimeout(post); + HttpPut put = new HttpPut(putUri); + configureRequestTimeout(put); - post.addHeader("Connection", "keep-alive"); + put.addHeader("Connection", "keep-alive"); + put.setEntity(new InputStreamEntity(fileInputStream, contentLength, ContentType.APPLICATION_OCTET_STREAM)); - MultipartEntityBuilder builder = MultipartEntityBuilder.create(); - builder.addBinaryBody( - "file", - fileIs, - ContentType.APPLICATION_OCTET_STREAM, - fileName - ); - - HttpEntity multipart = builder.build(); - post.setEntity(multipart); - - try (CloseableHttpResponse response = httpclient.execute(post)) { + try (CloseableHttpResponse response = httpclient.execute(put)) { int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NO_CONTENT) { LOGGER.log(Level.INFO, "Response Received. - Status OK"); @@ -381,7 +387,7 @@ class CTCloudHttpClient { HttpClientBuilder builder; if (ProxySettings.getProxyType() != ProxySettings.DIRECT_CONNECTION - && StringUtils.isBlank(ProxySettings.getAuthenticationUsername()) + && StringUtils.isBlank(ProxySettings.getAuthenticationUsername()) && ArrayUtils.isEmpty(ProxySettings.getAuthenticationPassword()) && WinHttpClients.isWinAuthAvailable()) { diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java index 6a093fee8e..f137818346 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java @@ -34,6 +34,9 @@ public class AuthTokenRequest { @JsonProperty("requestFileUpload") private boolean requestFileUpload; + @JsonProperty("fileUploadSize") + private Long fileUploadSize; + @JsonProperty("host_id") private String hostId; @@ -64,6 +67,16 @@ public class AuthTokenRequest { return this; } + public Long getFileUploadSize() { + return fileUploadSize; + } + + public AuthTokenRequest setFileUploadSize(Long fileUploadSize) { + this.fileUploadSize = fileUploadSize; + return this; + } + + public String getHostId() { return hostId; } diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java new file mode 100644 index 0000000000..48d8dc77b4 --- /dev/null +++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java @@ -0,0 +1,69 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2023 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.basistech.df.cybertriage.autopsy.ctapi.json; + +import java.io.InputStream; + +/** + * Data for a file upload request. + */ +public class FileUploadRequest { + + private String fullUrlPath; + private String fileName; + private InputStream fileInputStream; + private Long contentLength; + + public String getFullUrlPath() { + return fullUrlPath; + } + + public FileUploadRequest setFullUrlPath(String fullUrlPath) { + this.fullUrlPath = fullUrlPath; + return this; + } + + public String getFileName() { + return fileName; + } + + public FileUploadRequest setFileName(String fileName) { + this.fileName = fileName; + return this; + } + + public InputStream getFileInputStream() { + return fileInputStream; + } + + public FileUploadRequest setFileInputStream(InputStream fileInputStream) { + this.fileInputStream = fileInputStream; + return this; + } + + public Long getContentLength() { + return contentLength; + } + + public FileUploadRequest setContentLength(Long contentLength) { + this.contentLength = contentLength; + return this; + } + +} diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java index 2d27671b29..ba8bd556b1 100644 --- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java +++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java @@ -23,6 +23,7 @@ import com.basistech.df.cybertriage.autopsy.ctapi.CTCloudException; import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthTokenResponse; import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthenticatedRequestData; import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean; +import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest; import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo; import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status; import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest; @@ -112,8 +113,10 @@ class MalwareScanIngestModule implements FileIngestModule { //minimum file uploads left before issuing warning private static final long LOW_UPLOADS_REMAINING = 25; + // min and max upload size in bytes private static final long MIN_UPLOAD_SIZE = 1; - private static final long MAX_UPLOAD_SIZE = 1_000_000_000; + private static final long MAX_UPLOAD_SIZE = 100_000_000; // 100MB + private static final int NUM_FILE_UPLOAD_RETRIES = 7; private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000; @@ -640,7 +643,7 @@ class MalwareScanIngestModule implements FileIngestModule { } // get auth token / file upload url - AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), true); + AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), af.getSize()); if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) { throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR); } else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) { @@ -658,7 +661,13 @@ class MalwareScanIngestModule implements FileIngestModule { // upload bytes ReadContentInputStream fileInputStream = new ReadContentInputStream(af); - ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream); + + ctApiDAO.uploadFile(new FileUploadRequest() + .setContentLength(af.getSize()) + .setFileInputStream(fileInputStream) + .setFileName(af.getName()) + .setFullUrlPath(authTokenResponse.getFileUploadUrl()) + ); // upload metadata MetadataUploadRequest metaRequest = new MetadataUploadRequest()