From 69eb3e6d7febb936349102347edb32f42ed512da Mon Sep 17 00:00:00 2001 From: millmanorama Date: Fri, 9 Nov 2018 11:44:20 +0100 Subject: [PATCH 1/7] first pass at FileTypeUtils --- .../AbstractCommonAttributeSearcher.java | 11 +- .../autopsy/coreutils/FileTypeUtils.java | 225 ++++++++++++++++++ .../autopsy/timeline/utils/FilterUtils.java | 67 +----- 3 files changed, 242 insertions(+), 61 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java index 2e95e83007..4636b58bcb 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java @@ -97,9 +97,10 @@ public abstract class AbstractCommonAttributeSearcher { /** * Get the portion of the title that will display the frequency percentage * threshold. Items that existed in over this percent of data sources were - * ommited from the results. + * ommited from the results. * - * @return A string providing the frequency percentage threshold, or an empty string if no threshold was set + * @return A string providing the frequency percentage threshold, or an + * empty string if no threshold was set */ @NbBundle.Messages({ "# {0} - threshold percent", @@ -171,15 +172,15 @@ public abstract class AbstractCommonAttributeSearcher { */ static final Set TEXT_FILES_MIME_TYPES = Stream.of( "text/plain", //NON-NLS - "application/rtf", //NON-NLS - "application/pdf", //NON-NLS "text/css", //NON-NLS "text/html", //NON-NLS "text/csv", //NON-NLS + "application/rtf", //NON-NLS + "application/pdf", //NON-NLS + "text/calendar", //NON-NLS "application/json", //NON-NLS "application/javascript", //NON-NLS "application/xml", //NON-NLS - "text/calendar", //NON-NLS "application/x-msoffice", //NON-NLS "application/x-ooxml", //NON-NLS "application/msword", //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java new file mode 100644 index 0000000000..35b4866f1d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java @@ -0,0 +1,225 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.coreutils; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableSet; +import java.util.Arrays; +import static java.util.Arrays.asList; +import java.util.Collection; +import java.util.Collections; +import javax.imageio.ImageIO; +import org.apache.commons.collections4.CollectionUtils; +import org.openide.util.NbBundle; + +/** + * Utilities for dealing with file/mime-types + */ +public final class FileTypeUtils { + + private static final ImmutableCollection IMAGE_MIME_TYPES; + private static final ImmutableCollection AUDIO_MIME_TYPES; + private static final ImmutableCollection VIDEO_MIME_TYPES; + private static final ImmutableCollection MEDIA_MIME_TYPES; + private static final ImmutableCollection DOCUMENT_MIME_TYPES; + private static final ImmutableCollection EXECUTABLE_MIME_TYPES; + + static { + IMAGE_MIME_TYPES = new ImmutableSet.Builder() + .addAll(asList(ImageIO.getReaderMIMETypes())) + .add("image/bmp", //NON-NLS + "image/gif", //NON-NLS + "image/jpeg", //NON-NLS + "image/png", //NON-NLS + "image/tiff", //NON-NLS + "image/vnd.adobe.photoshop", //NON-NLS + "image/x-raw-nikon", //NON-NLS + "image/x-ms-bmp", //NON-NLS + "image/x-icon" //NON-NLS + ).build(); + AUDIO_MIME_TYPES = new ImmutableSet.Builder() + .add("image/bmp", //NON-NLS + "image/gif", //NON-NLS + "image/jpeg", //NON-NLS + "image/png", //NON-NLS + "image/tiff", //NON-NLS + "image/vnd.adobe.photoshop", //NON-NLS + "image/x-raw-nikon", //NON-NLS + "image/x-ms-bmp", //NON-NLS + "image/x-icon") //NON-NLS + .build(); + VIDEO_MIME_TYPES = new ImmutableSet.Builder() + .add("video/webm", //NON-NLS + "video/3gpp", //NON-NLS + "video/3gpp2", //NON-NLS + "video/ogg", //NON-NLS + "video/mpeg", //NON-NLS + "video/mp4", //NON-NLS + "video/quicktime", //NON-NLS + "video/x-msvideo", //NON-NLS + "video/x-flv", //NON-NLS + "video/x-m4v", //NON-NLS + "video/x-ms-wmv") + .build(); + MEDIA_MIME_TYPES = new ImmutableSet.Builder() + .addAll(IMAGE_MIME_TYPES) + .addAll(AUDIO_MIME_TYPES) + .addAll(VIDEO_MIME_TYPES) + .build(); + DOCUMENT_MIME_TYPES = new ImmutableSet.Builder() + .add("text/plain", //NON-NLS + "text/css", //NON-NLS + "text/html", //NON-NLS + "text/csv", //NON-NLS + "application/rtf", //NON-NLS + "application/pdf", //NON-NLS + "application/json", //NON-NLS + "application/javascript", //NON-NLS + "application/xml", //NON-NLS + "application/x-msoffice", //NON-NLS + "application/x-ooxml", //NON-NLS + "application/msword", //NON-NLS + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS + "application/vnd.ms-powerpoint", //NON-NLS + "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS + "application/vnd.ms-excel", //NON-NLS + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS + "application/vnd.oasis.opendocument.presentation", //NON-NLS + "application/vnd.oasis.opendocument.spreadsheet", //NON-NLS + "application/vnd.oasis.opendocument.text" //NON-NLS) + ).build(); + EXECUTABLE_MIME_TYPES = new ImmutableSet.Builder() + .add("application/x-bat",//NON-NLS + "application/x-dosexec",//NON-NLS + "application/vnd.microsoft.portable-executable",//NON-NLS + "application/x-msdownload",//NON-NLS + "application/exe",//NON-NLS + "application/x-exe",//NON-NLS + "application/dos-exe",//NON-NLS + "vms/exe",//NON-NLS + "application/x-winexe",//NON-NLS + "application/msdos-windows",//NON-NLS + "application/x-msdos-program"//NON-NLS + ).build(); + } + + private FileTypeUtils() { + + } + + /** + * Enum of categories/groups of file types. + */ + @NbBundle.Messages({ + "FileTypeUtils.FileTypeCategory.Audio.displayName=Audio", + "FileTypeUtils.FileTypeCategory.Video.displayName=Video", + "FileTypeUtils.FileTypeCategory.Image.displayName=Image", + "FileTypeUtils.FileTypeCategory.Media.displayName=Media", + "FileTypeUtils.FileTypeCategory.Documents.displayName=Documents", + "FileTypeUtils.FileTypeCategory.Executables.displayName=Executables"}) + public enum FileTypeCategory { + + IMAGE(Bundle.FileTypeUtils_FileTypeCategory_Image_displayName(), Collections.emptyList(), Collections.emptyList()), + VIDEO(Bundle.FileTypeUtils_FileTypeCategory_Video_displayName(), Collections.emptyList(), Collections.emptyList()), + AUDIO(Bundle.FileTypeUtils_FileTypeCategory_Audio_displayName(), Collections.emptyList(), Collections.emptyList()), + Media(Bundle.FileTypeUtils_FileTypeCategory_Media_displayName(), + CollectionUtils.union(Arrays.asList(ImageIO.getReaderMIMETypes()), + Arrays.asList( + "image/bmp", //NON-NLS + "image/gif", //NON-NLS + "image/jpeg", //NON-NLS + "image/png", //NON-NLS + "image/tiff", //NON-NLS + "image/vnd.adobe.photoshop", //NON-NLS + "image/x-raw-nikon", //NON-NLS + "image/x-ms-bmp", //NON-NLS + "image/x-icon", //NON-NLS + "video/webm", //NON-NLS + "video/3gpp", //NON-NLS + "video/3gpp2", //NON-NLS + "video/ogg", //NON-NLS + "video/mpeg", //NON-NLS + "video/mp4", //NON-NLS + "video/quicktime", //NON-NLS + "video/x-msvideo", //NON-NLS + "video/x-flv", //NON-NLS + "video/x-m4v", //NON-NLS + "video/x-ms-wmv", //NON-NLS + "application/vnd.ms-asf", //NON-NLS + "application/vnd.rn-realmedia", //NON-NLS + "application/x-shockwave-flash" //NON-NLS + )), Collections.emptyList()), + Executable(Bundle.FileTypeUtils_FileTypeCategory_Executables_displayName(), + Arrays.asList( + "application/x-bat",//NON-NLS + "application/x-dosexec",//NON-NLS + "application/vnd.microsoft.portable-executable",//NON-NLS + "application/x-msdownload",//NON-NLS + "application/exe",//NON-NLS + "application/x-exe",//NON-NLS + "application/dos-exe",//NON-NLS + "vms/exe",//NON-NLS + "application/x-winexe",//NON-NLS + "application/msdos-windows",//NON-NLS + "application/x-msdos-program"//NON-NLS + ), Collections.emptyList()), + Documents(Bundle.FileTypeUtils_FileTypeCategory_Documents_displayName(), + Arrays.asList("text/*", //NON-NLS + "application/rtf", //NON-NLS + "application/pdf", //NON-NLS + "application/json", //NON-NLS + "application/javascript", //NON-NLS + "application/xml", //NON-NLS + "application/x-msoffice", //NON-NLS + "application/x-ooxml", //NON-NLS + "application/msword", //NON-NLS + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS + "application/vnd.ms-powerpoint", //NON-NLS + "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS + "application/vnd.ms-excel", //NON-NLS + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS + "application/vnd.oasis.opendocument.presentation", //NON-NLS + "application/vnd.oasis.opendocument.spreadsheet", //NON-NLS + "application/vnd.oasis.opendocument.text" //NON-NLS) + ), Collections.emptyList()); + + private final String displayName; + private final ImmutableCollection mediaTypes; + private final ImmutableCollection extensions; + + private FileTypeCategory(String displayName, Collection mediaTypes, Collection extensions) { + this.displayName = displayName; + this.mediaTypes = ImmutableSet.copyOf(mediaTypes); + this.extensions = ImmutableSet.copyOf(extensions); + } + + public String getDisplayName() { + return displayName; + } + + public ImmutableCollection getMediaTypes() { + return mediaTypes; + + } + + public ImmutableCollection getExtension() { + return extensions; + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/timeline/utils/FilterUtils.java b/Core/src/org/sleuthkit/autopsy/timeline/utils/FilterUtils.java index 0e6ff39d35..e400208cf0 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/utils/FilterUtils.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/utils/FilterUtils.java @@ -22,9 +22,10 @@ import com.google.common.net.MediaType; import java.util.Collection; import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.openide.util.NbBundle; +import static org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory.Documents; +import static org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory.Executable; +import static org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory.Media; import org.sleuthkit.datamodel.TimelineManager; import org.sleuthkit.datamodel.timeline.TimelineFilter.FileTypeFilter; import org.sleuthkit.datamodel.timeline.TimelineFilter.FileTypesFilter; @@ -34,55 +35,12 @@ import org.sleuthkit.datamodel.timeline.TimelineFilter.FileTypesFilter; */ public final class FilterUtils { - private static final Set MEDIA_MIME_TYPES = Stream.of( - "image/*",//NON-NLS - "video/*",//NON-NLS - "audio/*",//NON-NLS - "application/vnd.ms-asf", //NON-NLS - "application/vnd.rn-realmedia", //NON-NLS - "application/x-shockwave-flash" //NON-NLS - ).map(MediaType::parse).collect(Collectors.toSet()); - - private static final Set EXECUTABLE_MIME_TYPES = Stream.of( - "application/x-bat",//NON-NLS - "application/x-dosexec",//NON-NLS - "application/vnd.microsoft.portable-executable",//NON-NLS - "application/x-msdownload",//NON-NLS - "application/exe",//NON-NLS - "application/x-exe",//NON-NLS - "application/dos-exe",//NON-NLS - "vms/exe",//NON-NLS - "application/x-winexe",//NON-NLS - "application/msdos-windows",//NON-NLS - "application/x-msdos-program"//NON-NLS - ).map(MediaType::parse).collect(Collectors.toSet()); - - private static final Set DOCUMENT_MIME_TYPES = Stream.of( - "text/*", //NON-NLS - "application/rtf", //NON-NLS - "application/pdf", //NON-NLS - "application/json", //NON-NLS - "application/javascript", //NON-NLS - "application/xml", //NON-NLS - "application/x-msoffice", //NON-NLS - "application/x-ooxml", //NON-NLS - "application/msword", //NON-NLS - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS - "application/vnd.ms-powerpoint", //NON-NLS - "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS - "application/vnd.ms-excel", //NON-NLS - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS - "application/vnd.oasis.opendocument.presentation", //NON-NLS - "application/vnd.oasis.opendocument.spreadsheet", //NON-NLS - "application/vnd.oasis.opendocument.text" //NON-NLS - ).map(MediaType::parse).collect(Collectors.toSet()); - - private static final Set NON_OTHER_MIME_TYPES = new HashSet<>(); + private static final Set NON_OTHER_MIME_TYPES = new HashSet<>(); static { - NON_OTHER_MIME_TYPES.addAll(MEDIA_MIME_TYPES); - NON_OTHER_MIME_TYPES.addAll(DOCUMENT_MIME_TYPES); - NON_OTHER_MIME_TYPES.addAll(EXECUTABLE_MIME_TYPES); + NON_OTHER_MIME_TYPES.addAll(Documents.getMediaTypes()); + NON_OTHER_MIME_TYPES.addAll(Executable.getMediaTypes()); + NON_OTHER_MIME_TYPES.addAll(Media.getMediaTypes()); } private FilterUtils() { @@ -95,16 +53,13 @@ public final class FilterUtils { * @return The new FileTypesFilter. */ @NbBundle.Messages({ - "FilterUtils.mediaFilter.displayName=Media", - "FilterUtils.documentsFilter.displayName=Documents", - "FilterUtils.executablesFilter.displayName=Executables", "FilterUtils.otherFilter.displayName=Other"}) public static FileTypesFilter createDefaultFileTypesFilter() { FileTypesFilter fileTypesFilter = new FileTypesFilter(); - fileTypesFilter.addSubFilter(new FileTypeFilter(Bundle.FilterUtils_mediaFilter_displayName(), MEDIA_MIME_TYPES)); - fileTypesFilter.addSubFilter(new FileTypeFilter(Bundle.FilterUtils_documentsFilter_displayName(), DOCUMENT_MIME_TYPES)); - fileTypesFilter.addSubFilter(new FileTypeFilter(Bundle.FilterUtils_executablesFilter_displayName(), EXECUTABLE_MIME_TYPES)); + fileTypesFilter.addSubFilter(new FileTypeFilter(Media.getDisplayName(), Media.getMediaTypes())); + fileTypesFilter.addSubFilter(new FileTypeFilter(Documents.getDisplayName(), Documents.getMediaTypes())); + fileTypesFilter.addSubFilter(new FileTypeFilter(Executable.getDisplayName(), Executable.getMediaTypes())); fileTypesFilter.addSubFilter(new InverseFileTypeFilter(Bundle.FilterUtils_otherFilter_displayName(), NON_OTHER_MIME_TYPES)); return fileTypesFilter; @@ -116,7 +71,7 @@ public final class FilterUtils { */ private static class InverseFileTypeFilter extends FileTypeFilter { - InverseFileTypeFilter(String displayName, Collection mediaTypes) { + InverseFileTypeFilter(String displayName, Collection mediaTypes) { super(displayName, mediaTypes); } From 72e3f14ae452b3bcb786bb9cab28b41e985ecdfe Mon Sep 17 00:00:00 2001 From: millmanorama Date: Fri, 9 Nov 2018 12:43:44 +0100 Subject: [PATCH 2/7] flesh out FileTypeCategory. Inline usage of FileTypeCategories. refactor InterCaseCommonAttributeSearcher.determineMimeTypeFilter() for readability. --- .../AbstractCommonAttributeSearcher.java | 80 ++-------- .../AllInterCaseCommonAttributeSearcher.java | 15 +- .../IntraCaseCommonAttributeSearcher.java | 63 +++----- ...ingleInterCaseCommonAttributeSearcher.java | 15 +- .../autopsy/coreutils/FileTypeUtils.java | 149 +++++++----------- .../autopsy/timeline/utils/FilterUtils.java | 19 ++- 6 files changed, 114 insertions(+), 227 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java index 4636b58bcb..1ef14d9ede 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java @@ -21,16 +21,15 @@ package org.sleuthkit.autopsy.commonfilesearch; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; +import org.sleuthkit.autopsy.coreutils.FileTypeUtils; import org.sleuthkit.datamodel.TskCoreException; /** @@ -131,69 +130,6 @@ public abstract class AbstractCommonAttributeSearcher { return instanceCollatedCommonFiles; } - /* - * The set of the MIME types that will be checked for extension mismatches - * when checkType is ONLY_MEDIA. ".jpg", ".jpeg", ".png", ".psd", ".nef", - * ".tiff", ".bmp", ".tec" ".aaf", ".3gp", ".asf", ".avi", ".m1v", ".m2v", - * //NON-NLS ".m4v", ".mp4", ".mov", ".mpeg", ".mpg", ".mpe", ".mp4", ".rm", - * ".wmv", ".mpv", ".flv", ".swf" - */ - static final Set MEDIA_PICS_VIDEO_MIME_TYPES = Stream.of( - "image/bmp", //NON-NLS - "image/gif", //NON-NLS - "image/jpeg", //NON-NLS - "image/png", //NON-NLS - "image/tiff", //NON-NLS - "image/vnd.adobe.photoshop", //NON-NLS - "image/x-raw-nikon", //NON-NLS - "image/x-ms-bmp", //NON-NLS - "image/x-icon", //NON-NLS - "video/webm", //NON-NLS - "video/3gpp", //NON-NLS - "video/3gpp2", //NON-NLS - "video/ogg", //NON-NLS - "video/mpeg", //NON-NLS - "video/mp4", //NON-NLS - "video/quicktime", //NON-NLS - "video/x-msvideo", //NON-NLS - "video/x-flv", //NON-NLS - "video/x-m4v", //NON-NLS - "video/x-ms-wmv", //NON-NLS - "application/vnd.ms-asf", //NON-NLS - "application/vnd.rn-realmedia", //NON-NLS - "application/x-shockwave-flash" //NON-NLS - ).collect(Collectors.toSet()); - - /* - * The set of the MIME types that will be checked for extension mismatches - * when checkType is ONLY_TEXT_FILES. ".doc", ".docx", ".odt", ".xls", - * ".xlsx", ".ppt", ".pptx" ".txt", ".rtf", ".log", ".text", ".xml" ".html", - * ".htm", ".css", ".js", ".php", ".aspx" ".pdf" - */ - static final Set TEXT_FILES_MIME_TYPES = Stream.of( - "text/plain", //NON-NLS - "text/css", //NON-NLS - "text/html", //NON-NLS - "text/csv", //NON-NLS - "application/rtf", //NON-NLS - "application/pdf", //NON-NLS - "text/calendar", //NON-NLS - "application/json", //NON-NLS - "application/javascript", //NON-NLS - "application/xml", //NON-NLS - "application/x-msoffice", //NON-NLS - "application/x-ooxml", //NON-NLS - "application/msword", //NON-NLS - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS - "application/vnd.ms-powerpoint", //NON-NLS - "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS - "application/vnd.ms-excel", //NON-NLS - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS - "application/vnd.oasis.opendocument.presentation", //NON-NLS - "application/vnd.oasis.opendocument.spreadsheet", //NON-NLS - "application/vnd.oasis.opendocument.text" //NON-NLS - ).collect(Collectors.toSet()); - /** * @return the filterByMedia */ @@ -221,4 +157,16 @@ public abstract class AbstractCommonAttributeSearcher { void setFilterByDoc(boolean filterByDoc) { this.filterByDoc = filterByDoc; } + + + Set getMimeTypesToFilterOn() { + Set mimeTypesToFilterOn = new HashSet<>(); + if (isFilterByMedia()) { + mimeTypesToFilterOn.addAll(FileTypeUtils.FileTypeCategory.VISUAL.getMediaTypes()); + } + if (isFilterByDoc()) { + mimeTypesToFilterOn.addAll(FileTypeUtils.FileTypeCategory.DOCUMENTS.getMediaTypes()); + } + return mimeTypesToFilterOn; + } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java index ddcf19b09e..208ad812b1 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java @@ -26,10 +26,10 @@ import java.util.Set; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; -import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type; -import static org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher.MEDIA_PICS_VIDEO_MIME_TYPES; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; +import org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory; +import org.sleuthkit.datamodel.TskCoreException; /** * Algorithm which finds files anywhere in the Central Repo which also occur in @@ -57,14 +57,7 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut public CommonAttributeSearchResults findMatches() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException { InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType); Map interCaseCommonFiles = eamDbAttrInst.findInterCaseCommonAttributeValues(Case.getCurrentCase()); - - Set mimeTypesToFilterOn = new HashSet<>(); - if (isFilterByMedia()) { - mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); - } - if (isFilterByDoc()) { - mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); - } + Set mimeTypesToFilterOn = getMimeTypesToFilterOn(); return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java index ac8a009b11..ff4f55565b 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java @@ -1,16 +1,16 @@ /* - * + * * Autopsy Forensic Browser - * + * * Copyright 2018 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,10 +23,10 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.datamodel.HashUtility; @@ -45,25 +45,22 @@ import org.sleuthkit.datamodel.TskCoreException; @SuppressWarnings("PMD.AbstractNaming") public abstract class IntraCaseCommonAttributeSearcher extends AbstractCommonAttributeSearcher { - private static final String FILTER_BY_MIME_TYPES_WHERE_CLAUSE = " and mime_type in (%s)"; //NON-NLS // where %s is csv list of mime_types to filter on - private final Map dataSourceIdToNameMap; - + /** * Subclass this to implement different algorithms for getting common files. * - * @param dataSourceIdMap a map of obj_id to datasource name + * @param dataSourceIdMap a map of obj_id to datasource name * @param filterByMediaMimeType match only on files whose mime types can be - * broadly categorized as media types - * @param filterByDocMimeType match only on files whose mime types can be - * broadly categorized as document types + * broadly categorized as media types + * @param filterByDocMimeType match only on files whose mime types can be + * broadly categorized as document types */ IntraCaseCommonAttributeSearcher(Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType, int percentageThreshold) { super(filterByMediaMimeType, filterByDocMimeType, percentageThreshold); this.dataSourceIdToNameMap = dataSourceIdMap; } - - + Map getDataSourceIdToNameMap() { return Collections.unmodifiableMap(this.dataSourceIdToNameMap); } @@ -96,7 +93,8 @@ public abstract class IntraCaseCommonAttributeSearcher extends AbstractCommonAtt * tree table tab to the top component. * * @return a data object with all of the matched files in a hierarchical - * format + * format + * * @throws TskCoreException * @throws NoCurrentCaseException * @throws SQLException @@ -149,29 +147,18 @@ public abstract class IntraCaseCommonAttributeSearcher extends AbstractCommonAtt * expression will be conjoined to base query with an AND operator. * * @return sql fragment of the form: 'and "mime_type" in ( [comma delimited - * list of mime types] )' or empty string in the event that no types to - * filter on were given. + * list of mime types] )' or empty string in the event that no types + * to filter on were given. */ String determineMimeTypeFilter() { - - Set mimeTypesToFilterOn = new HashSet<>(); - String mimeTypeString = ""; - if (isFilterByMedia()) { - mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); + Set mimeTypesToFilterOn = getMimeTypesToFilterOn(); + if (mimeTypesToFilterOn.isEmpty()) { + return ""; + } else { + String mimeTypeString = mimeTypesToFilterOn.stream() + .map(s -> "'" + s + "'") + .collect(Collectors.joining(",")); + return String.format(" and mime_type in (%s)", new Object[]{mimeTypeString}); } - if (isFilterByDoc()) { - mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); - } - StringBuilder mimeTypeFilter = new StringBuilder(mimeTypesToFilterOn.size()); - if (!mimeTypesToFilterOn.isEmpty()) { - for (String mimeType : mimeTypesToFilterOn) { - mimeTypeFilter.append(SINGLE_QUOTE).append(mimeType).append(SINGLE_QUTOE_COMMA); - } - mimeTypeString = mimeTypeFilter.toString().substring(0, mimeTypeFilter.length() - 1); - mimeTypeString = String.format(FILTER_BY_MIME_TYPES_WHERE_CLAUSE, new Object[]{mimeTypeString}); - } - return mimeTypeString; } - static final String SINGLE_QUTOE_COMMA = "',"; - static final String SINGLE_QUOTE = "'"; } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java index f7c87abdff..ed57fb8d10 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java @@ -20,17 +20,15 @@ package org.sleuthkit.autopsy.commonfilesearch; import java.sql.SQLException; -import java.util.HashSet; import java.util.Map; import java.util.Set; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type; -import static org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher.MEDIA_PICS_VIDEO_MIME_TYPES; /** * @@ -52,7 +50,7 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri * @throws EamDbException */ public SingleInterCaseCommonAttributeSearcher(int correlationCaseId, boolean filterByMediaMimeType, - boolean filterByDocMimeType, Type corAttrType, int percentageThreshold) throws EamDbException { + boolean filterByDocMimeType, Type corAttrType, int percentageThreshold) throws EamDbException { super(filterByMediaMimeType, filterByDocMimeType, corAttrType, percentageThreshold); this.corrleationCaseId = correlationCaseId; @@ -81,16 +79,11 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri CommonAttributeSearchResults findFiles(CorrelationCase correlationCase) throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException { InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType); Map interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseCommonAttributeValues(Case.getCurrentCase(), correlationCase); - Set mimeTypesToFilterOn = new HashSet<>(); - if (isFilterByMedia()) { - mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); - } - if (isFilterByDoc()) { - mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); - } + Set mimeTypesToFilterOn = getMimeTypesToFilterOn(); return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); } + @NbBundle.Messages({ "# {0} - case name", "# {1} - attr type", diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java index 35b4866f1d..53159afa1f 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java @@ -20,12 +20,10 @@ package org.sleuthkit.autopsy.coreutils; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableSet; -import java.util.Arrays; import static java.util.Arrays.asList; import java.util.Collection; import java.util.Collections; import javax.imageio.ImageIO; -import org.apache.commons.collections4.CollectionUtils; import org.openide.util.NbBundle; /** @@ -33,12 +31,13 @@ import org.openide.util.NbBundle; */ public final class FileTypeUtils { - private static final ImmutableCollection IMAGE_MIME_TYPES; - private static final ImmutableCollection AUDIO_MIME_TYPES; - private static final ImmutableCollection VIDEO_MIME_TYPES; - private static final ImmutableCollection MEDIA_MIME_TYPES; - private static final ImmutableCollection DOCUMENT_MIME_TYPES; - private static final ImmutableCollection EXECUTABLE_MIME_TYPES; + private static final ImmutableSet IMAGE_MIME_TYPES; + private static final ImmutableSet AUDIO_MIME_TYPES; + private static final ImmutableSet VIDEO_MIME_TYPES; + private static final ImmutableSet MULTI_MEDIA_MIME_TYPES; + private static final ImmutableSet DOCUMENT_MIME_TYPES; + private static final ImmutableSet EXECUTABLE_MIME_TYPES; + private static final ImmutableSet VISUAL_MEDIA_MIME_TYPES; static { IMAGE_MIME_TYPES = new ImmutableSet.Builder() @@ -51,19 +50,17 @@ public final class FileTypeUtils { "image/vnd.adobe.photoshop", //NON-NLS "image/x-raw-nikon", //NON-NLS "image/x-ms-bmp", //NON-NLS - "image/x-icon" //NON-NLS + "image/x-icon", //NON-NLS + "image/webp", //NON-NLS + "image/vnd.microsoft.icon" //NON-NLS ).build(); AUDIO_MIME_TYPES = new ImmutableSet.Builder() - .add("image/bmp", //NON-NLS - "image/gif", //NON-NLS - "image/jpeg", //NON-NLS - "image/png", //NON-NLS - "image/tiff", //NON-NLS - "image/vnd.adobe.photoshop", //NON-NLS - "image/x-raw-nikon", //NON-NLS - "image/x-ms-bmp", //NON-NLS - "image/x-icon") //NON-NLS - .build(); + .add("audio/midi", //NON-NLS + "audio/mpeg", //NON-NLS + "audio/webm", //NON-NLS + "audio/ogg", //NON-NLS + "audio/wav" //NON-NLS + ).build(); VIDEO_MIME_TYPES = new ImmutableSet.Builder() .add("video/webm", //NON-NLS "video/3gpp", //NON-NLS @@ -75,12 +72,23 @@ public final class FileTypeUtils { "video/x-msvideo", //NON-NLS "video/x-flv", //NON-NLS "video/x-m4v", //NON-NLS - "video/x-ms-wmv") - .build(); - MEDIA_MIME_TYPES = new ImmutableSet.Builder() + "video/x-ms-wmv"//NON-NLS + ).build(); + VISUAL_MEDIA_MIME_TYPES = new ImmutableSet.Builder() + .addAll(IMAGE_MIME_TYPES) + .addAll(VIDEO_MIME_TYPES) + .add("application/vnd.ms-asf", //NON-NLS + "application/vnd.rn-realmedia", //NON-NLS + "application/x-shockwave-flash" //NON-NLS + ).build(); + MULTI_MEDIA_MIME_TYPES = new ImmutableSet.Builder() .addAll(IMAGE_MIME_TYPES) .addAll(AUDIO_MIME_TYPES) .addAll(VIDEO_MIME_TYPES) + .add("application/vnd.ms-asf", //NON-NLS + "application/vnd.rn-realmedia", //NON-NLS + "application/x-shockwave-flash" //NON-NLS + ) .build(); DOCUMENT_MIME_TYPES = new ImmutableSet.Builder() .add("text/plain", //NON-NLS @@ -102,7 +110,7 @@ public final class FileTypeUtils { "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS "application/vnd.oasis.opendocument.presentation", //NON-NLS "application/vnd.oasis.opendocument.spreadsheet", //NON-NLS - "application/vnd.oasis.opendocument.text" //NON-NLS) + "application/vnd.oasis.opendocument.text" //NON-NLS ).build(); EXECUTABLE_MIME_TYPES = new ImmutableSet.Builder() .add("application/x-bat",//NON-NLS @@ -131,77 +139,36 @@ public final class FileTypeUtils { "FileTypeUtils.FileTypeCategory.Video.displayName=Video", "FileTypeUtils.FileTypeCategory.Image.displayName=Image", "FileTypeUtils.FileTypeCategory.Media.displayName=Media", + "FileTypeUtils.FileTypeCategory.Visual.displayName=Visual", "FileTypeUtils.FileTypeCategory.Documents.displayName=Documents", "FileTypeUtils.FileTypeCategory.Executables.displayName=Executables"}) - public enum FileTypeCategory { + static public enum FileTypeCategory { - IMAGE(Bundle.FileTypeUtils_FileTypeCategory_Image_displayName(), Collections.emptyList(), Collections.emptyList()), - VIDEO(Bundle.FileTypeUtils_FileTypeCategory_Video_displayName(), Collections.emptyList(), Collections.emptyList()), - AUDIO(Bundle.FileTypeUtils_FileTypeCategory_Audio_displayName(), Collections.emptyList(), Collections.emptyList()), - Media(Bundle.FileTypeUtils_FileTypeCategory_Media_displayName(), - CollectionUtils.union(Arrays.asList(ImageIO.getReaderMIMETypes()), - Arrays.asList( - "image/bmp", //NON-NLS - "image/gif", //NON-NLS - "image/jpeg", //NON-NLS - "image/png", //NON-NLS - "image/tiff", //NON-NLS - "image/vnd.adobe.photoshop", //NON-NLS - "image/x-raw-nikon", //NON-NLS - "image/x-ms-bmp", //NON-NLS - "image/x-icon", //NON-NLS - "video/webm", //NON-NLS - "video/3gpp", //NON-NLS - "video/3gpp2", //NON-NLS - "video/ogg", //NON-NLS - "video/mpeg", //NON-NLS - "video/mp4", //NON-NLS - "video/quicktime", //NON-NLS - "video/x-msvideo", //NON-NLS - "video/x-flv", //NON-NLS - "video/x-m4v", //NON-NLS - "video/x-ms-wmv", //NON-NLS - "application/vnd.ms-asf", //NON-NLS - "application/vnd.rn-realmedia", //NON-NLS - "application/x-shockwave-flash" //NON-NLS - )), Collections.emptyList()), - Executable(Bundle.FileTypeUtils_FileTypeCategory_Executables_displayName(), - Arrays.asList( - "application/x-bat",//NON-NLS - "application/x-dosexec",//NON-NLS - "application/vnd.microsoft.portable-executable",//NON-NLS - "application/x-msdownload",//NON-NLS - "application/exe",//NON-NLS - "application/x-exe",//NON-NLS - "application/dos-exe",//NON-NLS - "vms/exe",//NON-NLS - "application/x-winexe",//NON-NLS - "application/msdos-windows",//NON-NLS - "application/x-msdos-program"//NON-NLS - ), Collections.emptyList()), - Documents(Bundle.FileTypeUtils_FileTypeCategory_Documents_displayName(), - Arrays.asList("text/*", //NON-NLS - "application/rtf", //NON-NLS - "application/pdf", //NON-NLS - "application/json", //NON-NLS - "application/javascript", //NON-NLS - "application/xml", //NON-NLS - "application/x-msoffice", //NON-NLS - "application/x-ooxml", //NON-NLS - "application/msword", //NON-NLS - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS - "application/vnd.ms-powerpoint", //NON-NLS - "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS - "application/vnd.ms-excel", //NON-NLS - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS - "application/vnd.oasis.opendocument.presentation", //NON-NLS - "application/vnd.oasis.opendocument.spreadsheet", //NON-NLS - "application/vnd.oasis.opendocument.text" //NON-NLS) - ), Collections.emptyList()); + IMAGE(Bundle.FileTypeUtils_FileTypeCategory_Image_displayName(), + IMAGE_MIME_TYPES, + Collections.emptyList()), + VIDEO(Bundle.FileTypeUtils_FileTypeCategory_Video_displayName(), + VIDEO_MIME_TYPES, + Collections.emptyList()), + AUDIO(Bundle.FileTypeUtils_FileTypeCategory_Audio_displayName(), + AUDIO_MIME_TYPES, + Collections.emptyList()), + VISUAL(Bundle.FileTypeUtils_FileTypeCategory_Media_displayName(), + VISUAL_MEDIA_MIME_TYPES, + Collections.emptyList()), + MEDIA(Bundle.FileTypeUtils_FileTypeCategory_Media_displayName(), + MULTI_MEDIA_MIME_TYPES, + Collections.emptyList()), + EXECUTABLE(Bundle.FileTypeUtils_FileTypeCategory_Executables_displayName(), + EXECUTABLE_MIME_TYPES, + Collections.emptyList()), + DOCUMENTS(Bundle.FileTypeUtils_FileTypeCategory_Documents_displayName(), + DOCUMENT_MIME_TYPES, + Collections.emptyList()); private final String displayName; - private final ImmutableCollection mediaTypes; - private final ImmutableCollection extensions; + private final ImmutableSet mediaTypes; + private final ImmutableSet extensions; private FileTypeCategory(String displayName, Collection mediaTypes, Collection extensions) { this.displayName = displayName; @@ -213,12 +180,12 @@ public final class FileTypeUtils { return displayName; } - public ImmutableCollection getMediaTypes() { + public ImmutableSet getMediaTypes() { return mediaTypes; } - public ImmutableCollection getExtension() { + public ImmutableSet getExtension() { return extensions; } } diff --git a/Core/src/org/sleuthkit/autopsy/timeline/utils/FilterUtils.java b/Core/src/org/sleuthkit/autopsy/timeline/utils/FilterUtils.java index e400208cf0..7c4b3fc34b 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/utils/FilterUtils.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/utils/FilterUtils.java @@ -18,17 +18,16 @@ */ package org.sleuthkit.autopsy.timeline.utils; -import com.google.common.net.MediaType; import java.util.Collection; import java.util.HashSet; import java.util.Set; import org.openide.util.NbBundle; -import static org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory.Documents; -import static org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory.Executable; -import static org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory.Media; import org.sleuthkit.datamodel.TimelineManager; import org.sleuthkit.datamodel.timeline.TimelineFilter.FileTypeFilter; import org.sleuthkit.datamodel.timeline.TimelineFilter.FileTypesFilter; +import static org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory.MEDIA; +import static org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory.EXECUTABLE; +import static org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory.DOCUMENTS; /** * Utilities to deal with TimelineFilters @@ -38,9 +37,9 @@ public final class FilterUtils { private static final Set NON_OTHER_MIME_TYPES = new HashSet<>(); static { - NON_OTHER_MIME_TYPES.addAll(Documents.getMediaTypes()); - NON_OTHER_MIME_TYPES.addAll(Executable.getMediaTypes()); - NON_OTHER_MIME_TYPES.addAll(Media.getMediaTypes()); + NON_OTHER_MIME_TYPES.addAll(DOCUMENTS.getMediaTypes()); + NON_OTHER_MIME_TYPES.addAll(EXECUTABLE.getMediaTypes()); + NON_OTHER_MIME_TYPES.addAll(MEDIA.getMediaTypes()); } private FilterUtils() { @@ -57,9 +56,9 @@ public final class FilterUtils { public static FileTypesFilter createDefaultFileTypesFilter() { FileTypesFilter fileTypesFilter = new FileTypesFilter(); - fileTypesFilter.addSubFilter(new FileTypeFilter(Media.getDisplayName(), Media.getMediaTypes())); - fileTypesFilter.addSubFilter(new FileTypeFilter(Documents.getDisplayName(), Documents.getMediaTypes())); - fileTypesFilter.addSubFilter(new FileTypeFilter(Executable.getDisplayName(), Executable.getMediaTypes())); + fileTypesFilter.addSubFilter(new FileTypeFilter(MEDIA.getDisplayName(), MEDIA.getMediaTypes())); + fileTypesFilter.addSubFilter(new FileTypeFilter(DOCUMENTS.getDisplayName(), DOCUMENTS.getMediaTypes())); + fileTypesFilter.addSubFilter(new FileTypeFilter(EXECUTABLE.getDisplayName(), EXECUTABLE.getMediaTypes())); fileTypesFilter.addSubFilter(new InverseFileTypeFilter(Bundle.FilterUtils_otherFilter_displayName(), NON_OTHER_MIME_TYPES)); return fileTypesFilter; From 95ac53a1e883e76056bf219f6c2cfc700fc3d655 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Fri, 9 Nov 2018 12:51:29 +0100 Subject: [PATCH 3/7] rearrange code a bit --- .../autopsy/coreutils/FileTypeUtils.java | 156 +++++++++--------- 1 file changed, 81 insertions(+), 75 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java index 53159afa1f..b36796c7b7 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.coreutils; -import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableSet; import static java.util.Arrays.asList; import java.util.Collection; @@ -31,49 +30,90 @@ import org.openide.util.NbBundle; */ public final class FileTypeUtils { - private static final ImmutableSet IMAGE_MIME_TYPES; - private static final ImmutableSet AUDIO_MIME_TYPES; - private static final ImmutableSet VIDEO_MIME_TYPES; + private static final ImmutableSet IMAGE_MIME_TYPES + = new ImmutableSet.Builder() + .addAll(asList(ImageIO.getReaderMIMETypes())) + .add("image/bmp", //NON-NLS + "image/gif", //NON-NLS + "image/jpeg", //NON-NLS + "image/png", //NON-NLS + "image/tiff", //NON-NLS + "image/vnd.adobe.photoshop", //NON-NLS + "image/x-raw-nikon", //NON-NLS + "image/x-ms-bmp", //NON-NLS + "image/x-icon", //NON-NLS + "image/webp", //NON-NLS + "image/vnd.microsoft.icon", //NON-NLS + "image/x-rgb", //NON-NLS + "image/x-ms-bmp", //NON-NLS + "image/x-portable-graymap", //NON-NLS + "image/x-portable-bitmap" //NON-NLS + ).build(); + private static final ImmutableSet AUDIO_MIME_TYPES + = new ImmutableSet.Builder() + .add("audio/midi", //NON-NLS + "audio/mpeg", //NON-NLS + "audio/webm", //NON-NLS + "audio/ogg", //NON-NLS + "audio/wav" //NON-NLS + ).build(); + private static final ImmutableSet VIDEO_MIME_TYPES + = new ImmutableSet.Builder() + .add("video/webm", //NON-NLS + "video/3gpp", //NON-NLS + "video/3gpp2", //NON-NLS + "video/ogg", //NON-NLS + "video/mpeg", //NON-NLS + "video/mp4", //NON-NLS + "video/quicktime", //NON-NLS + "video/x-msvideo", //NON-NLS + "video/x-flv", //NON-NLS + "video/x-m4v", //NON-NLS + "video/x-ms-wmv"//NON-NLS + ).build(); + + private static final ImmutableSet DOCUMENT_MIME_TYPES + = new ImmutableSet.Builder() + .add("text/plain", //NON-NLS + "text/css", //NON-NLS + "text/html", //NON-NLS + "text/csv", //NON-NLS + "application/rtf", //NON-NLS + "application/pdf", //NON-NLS + "application/json", //NON-NLS + "application/javascript", //NON-NLS + "application/xml", //NON-NLS + "application/x-msoffice", //NON-NLS + "application/x-ooxml", //NON-NLS + "application/msword", //NON-NLS + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS + "application/vnd.ms-powerpoint", //NON-NLS + "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS + "application/vnd.ms-excel", //NON-NLS + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS + "application/vnd.oasis.opendocument.presentation", //NON-NLS + "application/vnd.oasis.opendocument.spreadsheet", //NON-NLS + "application/vnd.oasis.opendocument.text" //NON-NLS + ).build(); + private static final ImmutableSet EXECUTABLE_MIME_TYPES + = new ImmutableSet.Builder() + .add("application/x-bat",//NON-NLS + "application/x-dosexec",//NON-NLS + "application/vnd.microsoft.portable-executable",//NON-NLS + "application/x-msdownload",//NON-NLS + "application/exe",//NON-NLS + "application/x-exe",//NON-NLS + "application/dos-exe",//NON-NLS + "vms/exe",//NON-NLS + "application/x-winexe",//NON-NLS + "application/msdos-windows",//NON-NLS + "application/x-msdos-program"//NON-NLS + ).build(); + private static final ImmutableSet MULTI_MEDIA_MIME_TYPES; - private static final ImmutableSet DOCUMENT_MIME_TYPES; - private static final ImmutableSet EXECUTABLE_MIME_TYPES; private static final ImmutableSet VISUAL_MEDIA_MIME_TYPES; static { - IMAGE_MIME_TYPES = new ImmutableSet.Builder() - .addAll(asList(ImageIO.getReaderMIMETypes())) - .add("image/bmp", //NON-NLS - "image/gif", //NON-NLS - "image/jpeg", //NON-NLS - "image/png", //NON-NLS - "image/tiff", //NON-NLS - "image/vnd.adobe.photoshop", //NON-NLS - "image/x-raw-nikon", //NON-NLS - "image/x-ms-bmp", //NON-NLS - "image/x-icon", //NON-NLS - "image/webp", //NON-NLS - "image/vnd.microsoft.icon" //NON-NLS - ).build(); - AUDIO_MIME_TYPES = new ImmutableSet.Builder() - .add("audio/midi", //NON-NLS - "audio/mpeg", //NON-NLS - "audio/webm", //NON-NLS - "audio/ogg", //NON-NLS - "audio/wav" //NON-NLS - ).build(); - VIDEO_MIME_TYPES = new ImmutableSet.Builder() - .add("video/webm", //NON-NLS - "video/3gpp", //NON-NLS - "video/3gpp2", //NON-NLS - "video/ogg", //NON-NLS - "video/mpeg", //NON-NLS - "video/mp4", //NON-NLS - "video/quicktime", //NON-NLS - "video/x-msvideo", //NON-NLS - "video/x-flv", //NON-NLS - "video/x-m4v", //NON-NLS - "video/x-ms-wmv"//NON-NLS - ).build(); VISUAL_MEDIA_MIME_TYPES = new ImmutableSet.Builder() .addAll(IMAGE_MIME_TYPES) .addAll(VIDEO_MIME_TYPES) @@ -90,41 +130,7 @@ public final class FileTypeUtils { "application/x-shockwave-flash" //NON-NLS ) .build(); - DOCUMENT_MIME_TYPES = new ImmutableSet.Builder() - .add("text/plain", //NON-NLS - "text/css", //NON-NLS - "text/html", //NON-NLS - "text/csv", //NON-NLS - "application/rtf", //NON-NLS - "application/pdf", //NON-NLS - "application/json", //NON-NLS - "application/javascript", //NON-NLS - "application/xml", //NON-NLS - "application/x-msoffice", //NON-NLS - "application/x-ooxml", //NON-NLS - "application/msword", //NON-NLS - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS - "application/vnd.ms-powerpoint", //NON-NLS - "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS - "application/vnd.ms-excel", //NON-NLS - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS - "application/vnd.oasis.opendocument.presentation", //NON-NLS - "application/vnd.oasis.opendocument.spreadsheet", //NON-NLS - "application/vnd.oasis.opendocument.text" //NON-NLS - ).build(); - EXECUTABLE_MIME_TYPES = new ImmutableSet.Builder() - .add("application/x-bat",//NON-NLS - "application/x-dosexec",//NON-NLS - "application/vnd.microsoft.portable-executable",//NON-NLS - "application/x-msdownload",//NON-NLS - "application/exe",//NON-NLS - "application/x-exe",//NON-NLS - "application/dos-exe",//NON-NLS - "vms/exe",//NON-NLS - "application/x-winexe",//NON-NLS - "application/msdos-windows",//NON-NLS - "application/x-msdos-program"//NON-NLS - ).build(); + } private FileTypeUtils() { From 663c32df3d59dc4a828e9e24897dc31d72374e4a Mon Sep 17 00:00:00 2001 From: millmanorama Date: Fri, 9 Nov 2018 14:08:29 +0100 Subject: [PATCH 4/7] add some more mimetypes --- .../autopsy/coreutils/FileTypeUtils.java | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java index b36796c7b7..364b0908c1 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java @@ -23,6 +23,7 @@ import static java.util.Arrays.asList; import java.util.Collection; import java.util.Collections; import javax.imageio.ImageIO; +import static org.apache.commons.collections4.ListUtils.removeAll; import org.openide.util.NbBundle; /** @@ -32,7 +33,8 @@ public final class FileTypeUtils { private static final ImmutableSet IMAGE_MIME_TYPES = new ImmutableSet.Builder() - .addAll(asList(ImageIO.getReaderMIMETypes())) + .addAll(removeAll(asList(ImageIO.getReaderMIMETypes()), + asList("application/octet-stream"))) //this claims to be supported, but is not really an image. .add("image/bmp", //NON-NLS "image/gif", //NON-NLS "image/jpeg", //NON-NLS @@ -46,6 +48,7 @@ public final class FileTypeUtils { "image/vnd.microsoft.icon", //NON-NLS "image/x-rgb", //NON-NLS "image/x-ms-bmp", //NON-NLS + "image/x-xbitmap", //NON-NLS "image/x-portable-graymap", //NON-NLS "image/x-portable-bitmap" //NON-NLS ).build(); @@ -55,7 +58,9 @@ public final class FileTypeUtils { "audio/mpeg", //NON-NLS "audio/webm", //NON-NLS "audio/ogg", //NON-NLS - "audio/wav" //NON-NLS + "audio/wav", //NON-NLS + "audio/vnd.wave", //NON-NLS + "audio/x-ms-wma"//NON-NLS ).build(); private static final ImmutableSet VIDEO_MIME_TYPES = new ImmutableSet.Builder() @@ -71,13 +76,14 @@ public final class FileTypeUtils { "video/x-m4v", //NON-NLS "video/x-ms-wmv"//NON-NLS ).build(); - private static final ImmutableSet DOCUMENT_MIME_TYPES = new ImmutableSet.Builder() .add("text/plain", //NON-NLS "text/css", //NON-NLS "text/html", //NON-NLS "text/csv", //NON-NLS + "text/xml", //NON-NLS + "text/x-log", //NON-NLS "application/rtf", //NON-NLS "application/pdf", //NON-NLS "application/json", //NON-NLS @@ -86,10 +92,13 @@ public final class FileTypeUtils { "application/x-msoffice", //NON-NLS "application/x-ooxml", //NON-NLS "application/msword", //NON-NLS + "application/msword2", //NON-NLS + "application/vnd.wordperfect", //NON-NLS "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS "application/vnd.ms-powerpoint", //NON-NLS "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS "application/vnd.ms-excel", //NON-NLS + "application/vnd.ms-excel.sheet.4", //NON-NLS "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS "application/vnd.oasis.opendocument.presentation", //NON-NLS "application/vnd.oasis.opendocument.spreadsheet", //NON-NLS @@ -125,10 +134,6 @@ public final class FileTypeUtils { .addAll(IMAGE_MIME_TYPES) .addAll(AUDIO_MIME_TYPES) .addAll(VIDEO_MIME_TYPES) - .add("application/vnd.ms-asf", //NON-NLS - "application/vnd.rn-realmedia", //NON-NLS - "application/x-shockwave-flash" //NON-NLS - ) .build(); } @@ -141,34 +146,43 @@ public final class FileTypeUtils { * Enum of categories/groups of file types. */ @NbBundle.Messages({ - "FileTypeUtils.FileTypeCategory.Audio.displayName=Audio", - "FileTypeUtils.FileTypeCategory.Video.displayName=Video", - "FileTypeUtils.FileTypeCategory.Image.displayName=Image", - "FileTypeUtils.FileTypeCategory.Media.displayName=Media", - "FileTypeUtils.FileTypeCategory.Visual.displayName=Visual", - "FileTypeUtils.FileTypeCategory.Documents.displayName=Documents", - "FileTypeUtils.FileTypeCategory.Executables.displayName=Executables"}) + "FileTypeCategory.Audio.displayName=Audio", + "FileTypeCategory.Video.displayName=Video", + "FileTypeCategory.Image.displayName=Image", + "FileTypeCategory.Media.displayName=Media", + "FileTypeCategory.Visual.displayName=Visual", + "FileTypeCategory.Documents.displayName=Documents", + "FileTypeCategory.Executables.displayName=Executables"}) static public enum FileTypeCategory { - IMAGE(Bundle.FileTypeUtils_FileTypeCategory_Image_displayName(), + IMAGE(Bundle.FileTypeCategory_Image_displayName(), IMAGE_MIME_TYPES, Collections.emptyList()), - VIDEO(Bundle.FileTypeUtils_FileTypeCategory_Video_displayName(), + VIDEO(Bundle.FileTypeCategory_Video_displayName(), VIDEO_MIME_TYPES, Collections.emptyList()), - AUDIO(Bundle.FileTypeUtils_FileTypeCategory_Audio_displayName(), + AUDIO(Bundle.FileTypeCategory_Audio_displayName(), AUDIO_MIME_TYPES, Collections.emptyList()), - VISUAL(Bundle.FileTypeUtils_FileTypeCategory_Media_displayName(), + /** + * Images, Videos, flash Animations, etc + */ + VISUAL(Bundle.FileTypeCategory_Media_displayName(), VISUAL_MEDIA_MIME_TYPES, Collections.emptyList()), - MEDIA(Bundle.FileTypeUtils_FileTypeCategory_Media_displayName(), + /** + * VISUAL plus audio. + */ + MEDIA(Bundle.FileTypeCategory_Media_displayName(), MULTI_MEDIA_MIME_TYPES, Collections.emptyList()), - EXECUTABLE(Bundle.FileTypeUtils_FileTypeCategory_Executables_displayName(), + EXECUTABLE(Bundle.FileTypeCategory_Executables_displayName(), EXECUTABLE_MIME_TYPES, Collections.emptyList()), - DOCUMENTS(Bundle.FileTypeUtils_FileTypeCategory_Documents_displayName(), + /** + * (Plain) Text and "office" documents. + */ + DOCUMENTS(Bundle.FileTypeCategory_Documents_displayName(), DOCUMENT_MIME_TYPES, Collections.emptyList()); From 4f5a1159d10bf06a8df9cc9bc9eba0a094894647 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Thu, 15 Nov 2018 13:57:28 +0100 Subject: [PATCH 5/7] codacy fixes --- .../commonfilesearch/AllInterCaseCommonAttributeSearcher.java | 2 -- .../commonfilesearch/IntraCaseCommonAttributeSearcher.java | 3 +-- Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java index 208ad812b1..a4a4387ed0 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.commonfilesearch; import java.sql.SQLException; -import java.util.HashSet; import java.util.Map; import java.util.Set; import org.openide.util.NbBundle; @@ -28,7 +27,6 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; -import org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory; import org.sleuthkit.datamodel.TskCoreException; /** diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java index ff4f55565b..cc8990b518 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.datamodel.HashUtility; @@ -156,7 +155,7 @@ public abstract class IntraCaseCommonAttributeSearcher extends AbstractCommonAtt return ""; } else { String mimeTypeString = mimeTypesToFilterOn.stream() - .map(s -> "'" + s + "'") + .map(mimeType -> "'" + mimeType + "'") .collect(Collectors.joining(",")); return String.format(" and mime_type in (%s)", new Object[]{mimeTypeString}); } diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java index 364b0908c1..a5315718f0 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java @@ -153,7 +153,7 @@ public final class FileTypeUtils { "FileTypeCategory.Visual.displayName=Visual", "FileTypeCategory.Documents.displayName=Documents", "FileTypeCategory.Executables.displayName=Executables"}) - static public enum FileTypeCategory { + public enum FileTypeCategory { IMAGE(Bundle.FileTypeCategory_Image_displayName(), IMAGE_MIME_TYPES, From 6d78372a3bc24b6024fb12f7a30dfa331d491311 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Mon, 26 Nov 2018 15:46:09 +0100 Subject: [PATCH 6/7] remove static initialization block. minor cleanup. --- .../autopsy/coreutils/FileTypeUtils.java | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java index a5315718f0..3bf160cb47 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java @@ -118,25 +118,20 @@ public final class FileTypeUtils { "application/msdos-windows",//NON-NLS "application/x-msdos-program"//NON-NLS ).build(); - - private static final ImmutableSet MULTI_MEDIA_MIME_TYPES; - private static final ImmutableSet VISUAL_MEDIA_MIME_TYPES; - - static { - VISUAL_MEDIA_MIME_TYPES = new ImmutableSet.Builder() - .addAll(IMAGE_MIME_TYPES) - .addAll(VIDEO_MIME_TYPES) - .add("application/vnd.ms-asf", //NON-NLS - "application/vnd.rn-realmedia", //NON-NLS - "application/x-shockwave-flash" //NON-NLS - ).build(); - MULTI_MEDIA_MIME_TYPES = new ImmutableSet.Builder() - .addAll(IMAGE_MIME_TYPES) - .addAll(AUDIO_MIME_TYPES) - .addAll(VIDEO_MIME_TYPES) - .build(); - - } + private static final ImmutableSet MULTI_MEDIA_MIME_TYPES + = new ImmutableSet.Builder() + .addAll(IMAGE_MIME_TYPES) + .addAll(AUDIO_MIME_TYPES) + .addAll(VIDEO_MIME_TYPES) + .build(); + private static final ImmutableSet VISUAL_MEDIA_MIME_TYPES + = new ImmutableSet.Builder() + .addAll(IMAGE_MIME_TYPES) + .addAll(VIDEO_MIME_TYPES) + .add("application/vnd.ms-asf", //NON-NLS + "application/vnd.rn-realmedia", //NON-NLS + "application/x-shockwave-flash" //NON-NLS + ).build(); private FileTypeUtils() { @@ -153,7 +148,7 @@ public final class FileTypeUtils { "FileTypeCategory.Visual.displayName=Visual", "FileTypeCategory.Documents.displayName=Documents", "FileTypeCategory.Executables.displayName=Executables"}) - public enum FileTypeCategory { + public enum FileTypeCategory { IMAGE(Bundle.FileTypeCategory_Image_displayName(), IMAGE_MIME_TYPES, @@ -171,7 +166,7 @@ public final class FileTypeUtils { VISUAL_MEDIA_MIME_TYPES, Collections.emptyList()), /** - * VISUAL plus audio. + * VISUAL plus AUDIO. */ MEDIA(Bundle.FileTypeCategory_Media_displayName(), MULTI_MEDIA_MIME_TYPES, @@ -180,7 +175,7 @@ public final class FileTypeUtils { EXECUTABLE_MIME_TYPES, Collections.emptyList()), /** - * (Plain) Text and "office" documents. + * (Plain) Text and "Office" documents. */ DOCUMENTS(Bundle.FileTypeCategory_Documents_displayName(), DOCUMENT_MIME_TYPES, @@ -206,7 +201,7 @@ public final class FileTypeUtils { } public ImmutableSet getExtension() { - return extensions; + throw new UnsupportedOperationException("This method is not implemented yet."); //just to be explicit. } } } From fbb29d4e8de5717b0574b5e235f97e3a8d840f0c Mon Sep 17 00:00:00 2001 From: Jonathan Millman Date: Mon, 26 Nov 2018 16:17:25 +0100 Subject: [PATCH 7/7] fix type in method name --- Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java index 3bf160cb47..17c2061651 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/FileTypeUtils.java @@ -200,7 +200,7 @@ public final class FileTypeUtils { } - public ImmutableSet getExtension() { + public ImmutableSet getExtensions() { throw new UnsupportedOperationException("This method is not implemented yet."); //just to be explicit. } }