From 8593de7808edb8e3cb725abc2e582d065e6939da Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Tue, 26 Oct 2021 16:44:52 -0400 Subject: [PATCH 01/12] First cut --- .../mainui/datamodel/Bundle.properties-MERGED | 19 ++ .../datamodel/FileTypeSizeSearchParams.java | 1 + .../autopsy/mainui/datamodel/TagsDAO.java | 253 ++++++++++++++++++ .../mainui/datamodel/TagsSearchParams.java | 92 +++++++ 4 files changed, 365 insertions(+) create mode 100755 Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java create mode 100755 Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/Bundle.properties-MERGED index bbc9d5517b..ba656b0a66 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/Bundle.properties-MERGED @@ -46,7 +46,26 @@ FileExtRootFilter_documents_displayName=Documents FileExtRootFilter_executable_displayName=Executable FileExtRootFilter_image_displayName=Images FileExtRootFilter_video_displayName=Video +FileTag.name.text=File Tag FileTypesByMimeType.name.text=By MIME Type +TagsDAO.fileColumns.accessTimeColLbl=Accessed Time +TagsDAO.fileColumns.changeTimeColLbl=Changed Time +TagsDAO.fileColumns.commentColLbl=Comment +TagsDAO.fileColumns.createdTimeColLbl=Created Time +TagsDAO.fileColumns.filePathColLbl=File Path +TagsDAO.fileColumns.md5HashColLbl=MD5 Hash +TagsDAO.fileColumns.modifiedTimeColLbl=Modified Time +TagsDAO.fileColumns.nameColLbl=Name +TagsDAO.fileColumns.noDescription=No Description +TagsDAO.fileColumns.originalName=Original Name +TagsDAO.fileColumns.sizeColLbl=Size +TagsDAO.fileColumns.userNameColLbl=User Name +TagsDAO.tagColumns.commentColLbl=Comment +TagsDAO.tagColumns.origNameColLbl=Original Name +TagsDAO.tagColumns.sourceNameColLbl=Source Name +TagsDAO.tagColumns.sourcePathColLbl=Source File Path +TagsDAO.tagColumns.typeColLbl=Result Type +TagsDAO.tagColumns.userNameColLbl=User Name ThreePanelViewsDAO.fileColumns.accessTimeColLbl=Access Time ThreePanelViewsDAO.fileColumns.attrAddrColLbl=Attr. Addr. ThreePanelViewsDAO.fileColumns.changeTimeColLbl=Change Time diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java index 9cfa31744c..03b767373f 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java @@ -79,6 +79,7 @@ public class FileTypeSizeSearchParams extends BaseSearchParams { int hash = 7; hash = 23 * hash + Objects.hashCode(this.sizeFilter); hash = 23 * hash + Objects.hashCode(this.dataSourceId); + hash = 23 * hash + Objects.hashCode(super.hashCode()); return hash; } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java new file mode 100755 index 0000000000..3364a5ceb2 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java @@ -0,0 +1,253 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.mainui.datamodel; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.openide.util.NbBundle; +import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import static org.sleuthkit.autopsy.core.UserPreferences.hideKnownFilesInViewsTree; +import static org.sleuthkit.autopsy.core.UserPreferences.hideSlackFilesInViewsTree; +import org.sleuthkit.autopsy.coreutils.TimeZoneUtils; +import org.sleuthkit.autopsy.datamodel.FileTypeExtensions; +import org.sleuthkit.autopsy.mainui.datamodel.FileRowDTO.ExtensionMediaType; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; + +/** + * Provides information to populate the results viewer for data in the views + * section. + */ +@Messages({"TagsDAO.fileColumns.nameColLbl=Name", + "TagsDAO.fileColumns.originalName=Original Name", + "TagsDAO.fileColumns.filePathColLbl=File Path", + "TagsDAO.fileColumns.commentColLbl=Comment", + "TagsDAO.fileColumns.modifiedTimeColLbl=Modified Time", + "TagsDAO.fileColumns.changeTimeColLbl=Changed Time", + "TagsDAO.fileColumns.accessTimeColLbl=Accessed Time", + "TagsDAO.fileColumns.createdTimeColLbl=Created Time", + "TagsDAO.fileColumns.sizeColLbl=Size", + "TagsDAO.fileColumns.md5HashColLbl=MD5 Hash", + "TagsDAO.fileColumns.userNameColLbl=User Name", + "TagsDAO.fileColumns.noDescription=No Description", + "TagsDAO.tagColumns.sourceNameColLbl=Source Name", + "TagsDAO.tagColumns.origNameColLbl=Original Name", + "TagsDAO.tagColumns.sourcePathColLbl=Source File Path", + "TagsDAO.tagColumns.typeColLbl=Result Type", + "TagsDAO.tagColumns.commentColLbl=Comment", + "TagsDAO.tagColumns.userNameColLbl=User Name"}) +public class TagsDAO { + + private static final int CACHE_SIZE = 15; // rule of thumb: 5 entries times number of cached SearchParams sub-types + private static final long CACHE_DURATION = 2; + private static final TimeUnit CACHE_DURATION_UNITS = TimeUnit.MINUTES; + private final Cache searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); + + private static final String FILE_TAG_TYPE_ID = "FILE_TAG"; + private static final String RESULT_TAG_TYPE_ID = "RESULT_TAG"; + + private static final List FILE_TAG_COLUMNS = Arrays.asList( + getFileColumnKey(Bundle.TagsDAO_fileColumns_nameColLbl()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_originalName()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_filePathColLbl()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_commentColLbl()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_modifiedTimeColLbl()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_changeTimeColLbl()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_accessTimeColLbl()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_createdTimeColLbl()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_sizeColLbl()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_md5HashColLbl()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_userNameColLbl())); + + private static final List RESULT_TAG_COLUMNS = Arrays.asList( + getFileColumnKey(Bundle.TagsDAO_tagColumns_sourceNameColLbl()), + getFileColumnKey(Bundle.TagsDAO_tagColumns_origNameColLbl()), + getFileColumnKey(Bundle.TagsDAO_tagColumns_sourcePathColLbl()), + getFileColumnKey(Bundle.TagsDAO_tagColumns_typeColLbl()), + getFileColumnKey(Bundle.TagsDAO_tagColumns_commentColLbl()), + getFileColumnKey(Bundle.TagsDAO_tagColumns_userNameColLbl())); + + private static TagsDAO instance = null; + + synchronized static TagsDAO getInstance() { + if (instance == null) { + instance = new TagsDAO(); + } + + return instance; + } + + private static ColumnKey getFileColumnKey(String name) { + return new ColumnKey(name, name, Bundle.TagsDAO_fileColumns_noDescription()); + } + + private SleuthkitCase getCase() throws NoCurrentCaseException { + return Case.getCurrentCaseThrows().getSleuthkitCase(); + } + + static ExtensionMediaType getExtensionMediaType(String ext) { + if (StringUtils.isBlank(ext)) { + return ExtensionMediaType.UNCATEGORIZED; + } else { + ext = "." + ext; + } + if (FileTypeExtensions.getImageExtensions().contains(ext)) { + return ExtensionMediaType.IMAGE; + } else if (FileTypeExtensions.getVideoExtensions().contains(ext)) { + return ExtensionMediaType.VIDEO; + } else if (FileTypeExtensions.getAudioExtensions().contains(ext)) { + return ExtensionMediaType.AUDIO; + } else if (FileTypeExtensions.getDocumentExtensions().contains(ext)) { + return ExtensionMediaType.DOC; + } else if (FileTypeExtensions.getExecutableExtensions().contains(ext)) { + return ExtensionMediaType.EXECUTABLE; + } else if (FileTypeExtensions.getTextExtensions().contains(ext)) { + return ExtensionMediaType.TEXT; + } else if (FileTypeExtensions.getWebExtensions().contains(ext)) { + return ExtensionMediaType.WEB; + } else if (FileTypeExtensions.getPDFExtensions().contains(ext)) { + return ExtensionMediaType.PDF; + } else if (FileTypeExtensions.getArchiveExtensions().contains(ext)) { + return ExtensionMediaType.ARCHIVE; + } else { + return ExtensionMediaType.UNCATEGORIZED; + } + } + + public SearchResultsDTO getFilesByMime(FileTypeMimeSearchParams key) throws ExecutionException, IllegalArgumentException { + if (key.getMimeType() == null) { + throw new IllegalArgumentException("Must have non-null filter"); + } else if (key.getDataSourceId() != null && key.getDataSourceId() <= 0) { + throw new IllegalArgumentException("Data source id must be greater than 0 or null"); + } + + return searchParamsCache.get(key, () -> fetchMimeSearchResultsDTOs(key.getMimeType(), key.getDataSourceId(), key.getStartItem(), key.getMaxResultsCount())); + } + + private String getFileMimeWhereStatement(String mimeType, Long dataSourceId) { + + String whereClause = "(dir_type = " + TskData.TSK_FS_NAME_TYPE_ENUM.REG.getValue() + ")" + + " AND (type IN (" + + TskData.TSK_DB_FILES_TYPE_ENUM.FS.ordinal() + "," + + TskData.TSK_DB_FILES_TYPE_ENUM.CARVED.ordinal() + "," + + TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED.ordinal() + "," + + TskData.TSK_DB_FILES_TYPE_ENUM.LAYOUT_FILE.ordinal() + "," + + TskData.TSK_DB_FILES_TYPE_ENUM.LOCAL.ordinal() + + (hideSlackFilesInViewsTree() ? "" : ("," + TskData.TSK_DB_FILES_TYPE_ENUM.SLACK.ordinal())) + + "))" + + (dataSourceId != null && dataSourceId > 0 ? " AND data_source_obj_id = " + dataSourceId : " ") + + (hideKnownFilesInViewsTree() ? (" AND (known IS NULL OR known != " + TskData.FileKnown.KNOWN.getFileKnownValue() + ")") : "") + + " AND mime_type = '" + mimeType + "'"; + + return whereClause; + } + + @NbBundle.Messages({"FileTag.name.text=File Tag"}) + private SearchResultsDTO fetchMimeSearchResultsDTOs(String mimeType, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { + String whereStatement = getFileMimeWhereStatement(mimeType, dataSourceId); + final String FILE_TAG_DISPLAY_NAME = Bundle.FileTag_name_text(); + return fetchFileViewFiles(whereStatement, FILE_TAG_DISPLAY_NAME, startItem, maxResultCount); + } + + private SearchResultsDTO fetchFileViewFiles(String originalWhereStatement, String displayName, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { + + // Add offset and/or paging, if specified + String modifiedWhereStatement = originalWhereStatement + + " ORDER BY obj_id ASC" + + (maxResultCount != null && maxResultCount > 0 ? " LIMIT " + maxResultCount : "") + + (startItem > 0 ? " OFFSET " + startItem : ""); + + List files = getCase().findAllFilesWhere(modifiedWhereStatement); + + long totalResultsCount; + // get total number of results + if ( (startItem == 0) // offset is zero AND + && ( (maxResultCount != null && files.size() < maxResultCount) // number of results is less than max + || (maxResultCount == null)) ) { // OR max number of results was not specified + totalResultsCount = files.size(); + } else { + // do a query to get total number of results + totalResultsCount = getCase().countFilesWhere(originalWhereStatement); + } + + List fileRows = new ArrayList<>(); + for (AbstractFile file : files) { + + List cellValues = Arrays.asList( + file.getName(), // GVDTODO handle . and .. from getContentDisplayName() + // GVDTODO translation column + null, + //GVDTDO replace nulls with SCO + null, + null, + null, + file.getUniquePath(), + TimeZoneUtils.getFormattedTime(file.getMtime()), + TimeZoneUtils.getFormattedTime(file.getCtime()), + TimeZoneUtils.getFormattedTime(file.getAtime()), + TimeZoneUtils.getFormattedTime(file.getCrtime()), + file.getSize(), + file.getDirFlagAsString(), + file.getMetaFlagsAsString(), + // mode, + // userid, + // groupid, + // metaAddr, + // attrAddr, + // typeDir, + // typeMeta, + + file.getKnown().getName(), + StringUtils.defaultString(file.getMd5Hash()), + StringUtils.defaultString(file.getSha256Hash()), + // objectId, + + StringUtils.defaultString(file.getMIMEType()), + file.getNameExtension() + ); + + fileRows.add(new FileRowDTO( + file, + file.getId(), + file.getName(), + file.getNameExtension(), + getExtensionMediaType(file.getNameExtension()), + file.isDirNameFlagSet(TskData.TSK_FS_NAME_FLAG_ENUM.ALLOC), + file.getType(), + cellValues)); + } + + return new BaseSearchResultsDTO(FILE_TAG_TYPE_ID, displayName, FILE_TAG_COLUMNS, fileRows, startItem, totalResultsCount); + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java new file mode 100755 index 0000000000..6baa78aa8c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java @@ -0,0 +1,92 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.mainui.datamodel; + +import java.util.Objects; + +/** + * Key for accessing data about tags from the DAO. + */ +public class TagsSearchParams extends BaseSearchParams { + + public enum TagType { + FILE, + RESULT; + } + + private final TagType type; + private final String tagName; + private final Long dataSourceId; + + public TagsSearchParams(String tagName, TagType type, Long dataSourceId) { + this.tagName = tagName; + this.type = type; + this.dataSourceId = dataSourceId; + } + + public TagsSearchParams(String tagName, TagType type, Long dataSourceId, long startItem, Long maxResultsCount) { + super(startItem, maxResultsCount); + this.tagName = tagName; + this.type = type; + this.dataSourceId = dataSourceId; + } + + public String getMimeType() { + return tagName; + } + + public Long getDataSourceId() { + return dataSourceId; + } + + @Override + public int hashCode() { + int hash = 7; + hash = 23 * hash + Objects.hashCode(this.tagName); + hash = 23 * hash + Objects.hashCode(this.type); + hash = 23 * hash + Objects.hashCode(this.dataSourceId); + hash = 23 * hash + Objects.hashCode(super.hashCode()); + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final TagsSearchParams other = (TagsSearchParams) obj; + if (!(this.tagName.equals(other.tagName))) { + return false; + } + if (!Objects.equals(this.dataSourceId, other.dataSourceId)) { + return false; + } + if (!Objects.equals(this.type, other.type)) { + return false; + } + return super.equalFields(other); + } + +} From e8f9ed9fbc2bf035d5127476c996ba8269722fb7 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 27 Oct 2021 13:20:01 -0400 Subject: [PATCH 02/12] Resolved built issues --- .../mainui/datamodel/FileTypeSizeSearchParams.java | 1 - .../sleuthkit/autopsy/mainui/datamodel/TagsDAO.java | 11 ++++++++--- .../autopsy/mainui/datamodel/TagsSearchParams.java | 6 ++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java index 91dcb1e1a7..bba60bcb7c 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java @@ -73,7 +73,6 @@ public class FileTypeSizeSearchParams { int hash = 7; hash = 23 * hash + Objects.hashCode(this.sizeFilter); hash = 23 * hash + Objects.hashCode(this.dataSourceId); - hash = 23 * hash + Objects.hashCode(super.hashCode()); return hash; } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java index 3364a5ceb2..4a422d3e2a 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java @@ -71,7 +71,7 @@ public class TagsDAO { private static final int CACHE_SIZE = 15; // rule of thumb: 5 entries times number of cached SearchParams sub-types private static final long CACHE_DURATION = 2; private static final TimeUnit CACHE_DURATION_UNITS = TimeUnit.MINUTES; - private final Cache searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); + private final Cache, SearchResultsDTO> searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); private static final String FILE_TAG_TYPE_ID = "FILE_TAG"; private static final String RESULT_TAG_TYPE_ID = "RESULT_TAG"; @@ -144,14 +144,19 @@ public class TagsDAO { } } - public SearchResultsDTO getFilesByMime(FileTypeMimeSearchParams key) throws ExecutionException, IllegalArgumentException { + public SearchResultsDTO getFilesByMime(FileTypeMimeSearchParams key, long startItem, Long maxCount, boolean hardRefresh) throws ExecutionException, IllegalArgumentException { if (key.getMimeType() == null) { throw new IllegalArgumentException("Must have non-null filter"); } else if (key.getDataSourceId() != null && key.getDataSourceId() <= 0) { throw new IllegalArgumentException("Data source id must be greater than 0 or null"); } + + SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); + if (hardRefresh) { + this.searchParamsCache.invalidate(searchParams); + } - return searchParamsCache.get(key, () -> fetchMimeSearchResultsDTOs(key.getMimeType(), key.getDataSourceId(), key.getStartItem(), key.getMaxResultsCount())); + return searchParamsCache.get(searchParams, () -> fetchMimeSearchResultsDTOs(key.getMimeType(), key.getDataSourceId(), startItem, maxCount)); } private String getFileMimeWhereStatement(String mimeType, Long dataSourceId) { diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java index 6baa78aa8c..80a97392a6 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java @@ -23,7 +23,7 @@ import java.util.Objects; /** * Key for accessing data about tags from the DAO. */ -public class TagsSearchParams extends BaseSearchParams { +public class TagsSearchParams { public enum TagType { FILE, @@ -41,7 +41,6 @@ public class TagsSearchParams extends BaseSearchParams { } public TagsSearchParams(String tagName, TagType type, Long dataSourceId, long startItem, Long maxResultsCount) { - super(startItem, maxResultsCount); this.tagName = tagName; this.type = type; this.dataSourceId = dataSourceId; @@ -61,7 +60,6 @@ public class TagsSearchParams extends BaseSearchParams { hash = 23 * hash + Objects.hashCode(this.tagName); hash = 23 * hash + Objects.hashCode(this.type); hash = 23 * hash + Objects.hashCode(this.dataSourceId); - hash = 23 * hash + Objects.hashCode(super.hashCode()); return hash; } @@ -86,7 +84,7 @@ public class TagsSearchParams extends BaseSearchParams { if (!Objects.equals(this.type, other.type)) { return false; } - return super.equalFields(other); + return true; } } From 94e8f893a9e9c59bca5ccaa0f14fd4378ea4387e Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 27 Oct 2021 17:00:45 -0400 Subject: [PATCH 03/12] More work --- .../autopsy/mainui/datamodel/TagsDAO.java | 221 ++++++++++-------- .../mainui/datamodel/TagsSearchParams.java | 19 +- 2 files changed, 136 insertions(+), 104 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java index 4a422d3e2a..d71f199191 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java @@ -22,27 +22,25 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import static org.sleuthkit.autopsy.core.UserPreferences.hideKnownFilesInViewsTree; -import static org.sleuthkit.autopsy.core.UserPreferences.hideSlackFilesInViewsTree; +import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.TimeZoneUtils; import org.sleuthkit.autopsy.datamodel.FileTypeExtensions; import org.sleuthkit.autopsy.mainui.datamodel.FileRowDTO.ExtensionMediaType; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifactTag; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskData; /** * Provides information to populate the results viewer for data in the views @@ -73,12 +71,16 @@ public class TagsDAO { private static final TimeUnit CACHE_DURATION_UNITS = TimeUnit.MINUTES; private final Cache, SearchResultsDTO> searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); + private static final String USER_NAME_PROPERTY = "user.name"; //NON-NLS + private static final String FILE_TAG_TYPE_ID = "FILE_TAG"; private static final String RESULT_TAG_TYPE_ID = "RESULT_TAG"; - + private static final String FILE_TAG_DISPLAY_NAME = "File Tag"; + private static final String RESULT_TAG_DISPLAY_NAME = "Result Tag"; + private static final List FILE_TAG_COLUMNS = Arrays.asList( getFileColumnKey(Bundle.TagsDAO_fileColumns_nameColLbl()), - getFileColumnKey(Bundle.TagsDAO_fileColumns_originalName()), + getFileColumnKey(Bundle.TagsDAO_fileColumns_originalName()), // ELODO handle translation getFileColumnKey(Bundle.TagsDAO_fileColumns_filePathColLbl()), getFileColumnKey(Bundle.TagsDAO_fileColumns_commentColLbl()), getFileColumnKey(Bundle.TagsDAO_fileColumns_modifiedTimeColLbl()), @@ -144,115 +146,146 @@ public class TagsDAO { } } - public SearchResultsDTO getFilesByMime(FileTypeMimeSearchParams key, long startItem, Long maxCount, boolean hardRefresh) throws ExecutionException, IllegalArgumentException { - if (key.getMimeType() == null) { + public SearchResultsDTO getTags(TagsSearchParams key, long startItem, Long maxCount, boolean hardRefresh) throws ExecutionException, IllegalArgumentException { + if (key.getTagName() == null) { throw new IllegalArgumentException("Must have non-null filter"); } else if (key.getDataSourceId() != null && key.getDataSourceId() <= 0) { throw new IllegalArgumentException("Data source id must be greater than 0 or null"); } - SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); + SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); if (hardRefresh) { this.searchParamsCache.invalidate(searchParams); } - return searchParamsCache.get(searchParams, () -> fetchMimeSearchResultsDTOs(key.getMimeType(), key.getDataSourceId(), startItem, maxCount)); - } - - private String getFileMimeWhereStatement(String mimeType, Long dataSourceId) { - - String whereClause = "(dir_type = " + TskData.TSK_FS_NAME_TYPE_ENUM.REG.getValue() + ")" - + " AND (type IN (" - + TskData.TSK_DB_FILES_TYPE_ENUM.FS.ordinal() + "," - + TskData.TSK_DB_FILES_TYPE_ENUM.CARVED.ordinal() + "," - + TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED.ordinal() + "," - + TskData.TSK_DB_FILES_TYPE_ENUM.LAYOUT_FILE.ordinal() + "," - + TskData.TSK_DB_FILES_TYPE_ENUM.LOCAL.ordinal() - + (hideSlackFilesInViewsTree() ? "" : ("," + TskData.TSK_DB_FILES_TYPE_ENUM.SLACK.ordinal())) - + "))" - + (dataSourceId != null && dataSourceId > 0 ? " AND data_source_obj_id = " + dataSourceId : " ") - + (hideKnownFilesInViewsTree() ? (" AND (known IS NULL OR known != " + TskData.FileKnown.KNOWN.getFileKnownValue() + ")") : "") - + " AND mime_type = '" + mimeType + "'"; - - return whereClause; + return searchParamsCache.get(searchParams, () -> fetchTagsDTOs(key.getTagName(), key.getTagType(), key.getDataSourceId(), startItem, maxCount)); } @NbBundle.Messages({"FileTag.name.text=File Tag"}) - private SearchResultsDTO fetchMimeSearchResultsDTOs(String mimeType, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { - String whereStatement = getFileMimeWhereStatement(mimeType, dataSourceId); - final String FILE_TAG_DISPLAY_NAME = Bundle.FileTag_name_text(); - return fetchFileViewFiles(whereStatement, FILE_TAG_DISPLAY_NAME, startItem, maxResultCount); + private SearchResultsDTO fetchTagsDTOs(TagName tagName, TagsSearchParams.TagType type, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { + if (null == type) { + // ELTODO throw? + return null; + } + + switch (type) { + case FILE: + return fetchFileTags(tagName, dataSourceId, startItem, maxResultCount); + case RESULT: + return fetchResultTags(tagName, dataSourceId, startItem, maxResultCount); + default: + // ELTODO throw? + return null; + } } + + /* GET RESULT TAGS + * BlackboardArtifactTagNodeFactory.createKeys(List tags) + * BlackboardArtifactTagNode.createSheet() + */ + + /* GET FILE TAGS + * ContentTagNodeFactory.createKeys(List tags) + * ContentTagNode.createSheet() + */ + + private SearchResultsDTO fetchResultTags(TagName tagName, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { - private SearchResultsDTO fetchFileViewFiles(String originalWhereStatement, String displayName, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { + // ELTODO startItem, maxResultCount - // Add offset and/or paging, if specified - String modifiedWhereStatement = originalWhereStatement - + " ORDER BY obj_id ASC" - + (maxResultCount != null && maxResultCount > 0 ? " LIMIT " + maxResultCount : "") - + (startItem > 0 ? " OFFSET " + startItem : ""); - - List files = getCase().findAllFilesWhere(modifiedWhereStatement); - - long totalResultsCount; - // get total number of results - if ( (startItem == 0) // offset is zero AND - && ( (maxResultCount != null && files.size() < maxResultCount) // number of results is less than max - || (maxResultCount == null)) ) { // OR max number of results was not specified - totalResultsCount = files.size(); + List tags = new ArrayList<>(); + // Use the blackboard artifact tags bearing the specified tag name as the tags. + List artifactTags = (dataSourceId != null && dataSourceId > 0) + ? Case.getCurrentCaseThrows().getServices().getTagsManager().getBlackboardArtifactTagsByTagName(tagName, dataSourceId) + : Case.getCurrentCaseThrows().getServices().getTagsManager().getBlackboardArtifactTagsByTagName(tagName); + if (UserPreferences.showOnlyCurrentUserTags()) { + String userName = System.getProperty(USER_NAME_PROPERTY); + for (BlackboardArtifactTag tag : artifactTags) { + if (userName.equals(tag.getUserName())) { + tags.add(tag); + } + } } else { - // do a query to get total number of results - totalResultsCount = getCase().countFilesWhere(originalWhereStatement); + tags.addAll(artifactTags); } List fileRows = new ArrayList<>(); - for (AbstractFile file : files) { + for (BlackboardArtifactTag tag : tags) { + String name = tag.getContent().getName(); // As a backup. + try { + name = tag.getArtifact().getShortDescription(); + } catch (TskCoreException ignore) { + // it's a WARNING, skip + } + String contentPath; + try { + contentPath = tag.getContent().getUniquePath(); + } catch (TskCoreException ex) { + contentPath = NbBundle.getMessage(this.getClass(), "BlackboardArtifactTagNode.createSheet.unavail.text"); + } + List cellValues = Arrays.asList( - file.getName(), // GVDTODO handle . and .. from getContentDisplayName() - // GVDTODO translation column - null, - //GVDTDO replace nulls with SCO - null, - null, - null, - file.getUniquePath(), - TimeZoneUtils.getFormattedTime(file.getMtime()), - TimeZoneUtils.getFormattedTime(file.getCtime()), - TimeZoneUtils.getFormattedTime(file.getAtime()), - TimeZoneUtils.getFormattedTime(file.getCrtime()), - file.getSize(), - file.getDirFlagAsString(), - file.getMetaFlagsAsString(), - // mode, - // userid, - // groupid, - // metaAddr, - // attrAddr, - // typeDir, - // typeMeta, + name, + null, // ELTODO translation column + contentPath, + tag.getArtifact().getDisplayName(), + tag.getComment(), + tag.getUserName()); - file.getKnown().getName(), - StringUtils.defaultString(file.getMd5Hash()), - StringUtils.defaultString(file.getSha256Hash()), - // objectId, - - StringUtils.defaultString(file.getMIMEType()), - file.getNameExtension() - ); - - fileRows.add(new FileRowDTO( - file, - file.getId(), - file.getName(), - file.getNameExtension(), - getExtensionMediaType(file.getNameExtension()), - file.isDirNameFlagSet(TskData.TSK_FS_NAME_FLAG_ENUM.ALLOC), - file.getType(), - cellValues)); + fileRows.add(new BaseRowDTO( + cellValues, + RESULT_TAG_TYPE_ID, + tag.getId())); } - return new BaseSearchResultsDTO(FILE_TAG_TYPE_ID, displayName, FILE_TAG_COLUMNS, fileRows, startItem, totalResultsCount); + return new BaseSearchResultsDTO(RESULT_TAG_TYPE_ID, RESULT_TAG_DISPLAY_NAME, RESULT_TAG_COLUMNS, fileRows, startItem, fileRows.size()); } + + private SearchResultsDTO fetchFileTags(TagName tagName, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { + // ELTODO startItem, maxResultCount + + List tags = new ArrayList<>(); + List contentTags = (dataSourceId != null && dataSourceId > 0) + ? Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByTagName(tagName, dataSourceId) + : Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByTagName(tagName); + if (UserPreferences.showOnlyCurrentUserTags()) { + String userName = System.getProperty(USER_NAME_PROPERTY); + for (ContentTag tag : contentTags) { + if (userName.equals(tag.getUserName())) { + tags.add(tag); + } + } + } else { + tags.addAll(contentTags); + } + + List fileRows = new ArrayList<>(); + for (ContentTag tag : tags) { + Content content = tag.getContent(); + String contentPath = content.getUniquePath(); + AbstractFile file = content instanceof AbstractFile ? (AbstractFile) content : null; + + List cellValues = Arrays.asList( + content.getName(), + null, // ELTODO translation column + contentPath, + tag.getComment(), + file != null ? TimeZoneUtils.getFormattedTime(file.getMtime()) : "", + file != null ? TimeZoneUtils.getFormattedTime(file.getCtime()) : "", + file != null ? TimeZoneUtils.getFormattedTime(file.getAtime()) : "", + file != null ? TimeZoneUtils.getFormattedTime(file.getCrtime()) : "", + content.getSize(), + file != null ? StringUtils.defaultString(file.getMd5Hash()) : "", + tag.getUserName()); + + fileRows.add(new BaseRowDTO( + cellValues, + FILE_TAG_TYPE_ID, + file.getId())); + } + + return new BaseSearchResultsDTO(FILE_TAG_TYPE_ID, FILE_TAG_DISPLAY_NAME, FILE_TAG_COLUMNS, fileRows, startItem, fileRows.size()); + } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java index 80a97392a6..2f239047cd 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsSearchParams.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.mainui.datamodel; import java.util.Objects; +import org.sleuthkit.datamodel.TagName; /** * Key for accessing data about tags from the DAO. @@ -31,25 +32,23 @@ public class TagsSearchParams { } private final TagType type; - private final String tagName; + private final TagName tagName; private final Long dataSourceId; - public TagsSearchParams(String tagName, TagType type, Long dataSourceId) { + public TagsSearchParams(TagName tagName, TagType type, Long dataSourceId) { this.tagName = tagName; this.type = type; this.dataSourceId = dataSourceId; } - public TagsSearchParams(String tagName, TagType type, Long dataSourceId, long startItem, Long maxResultsCount) { - this.tagName = tagName; - this.type = type; - this.dataSourceId = dataSourceId; - } - - public String getMimeType() { + public TagName getTagName() { return tagName; } + public TagType getTagType() { + return type; + } + public Long getDataSourceId() { return dataSourceId; } @@ -75,7 +74,7 @@ public class TagsSearchParams { return false; } final TagsSearchParams other = (TagsSearchParams) obj; - if (!(this.tagName.equals(other.tagName))) { + if (!Objects.equals(this.tagName, other.tagName)) { return false; } if (!Objects.equals(this.dataSourceId, other.dataSourceId)) { From 24cf8315b89435084a92dd79f97cc8debf0ec907 Mon Sep 17 00:00:00 2001 From: apriestman Date: Thu, 28 Oct 2021 15:08:41 -0400 Subject: [PATCH 04/12] Optimize artifact methods --- .../mainui/datamodel/AnalysisResultDAO.java | 31 +++---- .../datamodel/AnalysisResultSearchParam.java | 51 +---------- .../datamodel/BlackboardArtifactDAO.java | 35 ++++++++ .../BlackboardArtifactSearchParam.java | 72 ++++++++++++++++ .../mainui/datamodel/DataArtifactDAO.java | 32 +++---- .../datamodel/DataArtifactSearchParam.java | 50 +---------- .../mainui/datamodel/TableSearchTest.java | 86 +++++++++++++++++++ 7 files changed, 230 insertions(+), 127 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactSearchParam.java diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java index c499abbfd3..c044fa8587 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java @@ -107,29 +107,26 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { } // TODO We can probably combine all the caches at some point - private final Cache, AnalysisResultTableSearchResultsDTO> analysisResultCache = CacheBuilder.newBuilder().maximumSize(1000).build(); + private final Cache, AnalysisResultTableSearchResultsDTO> analysisResultCache = CacheBuilder.newBuilder().maximumSize(1000).build(); private final Cache, AnalysisResultTableSearchResultsDTO> hashHitCache = CacheBuilder.newBuilder().maximumSize(1000).build(); private final Cache, AnalysisResultTableSearchResultsDTO> keywordHitCache = CacheBuilder.newBuilder().maximumSize(1000).build(); - private AnalysisResultTableSearchResultsDTO fetchAnalysisResultsForTable(SearchParams cacheKey) throws NoCurrentCaseException, TskCoreException { + private AnalysisResultTableSearchResultsDTO fetchAnalysisResultsForTable(SearchParams cacheKey) throws NoCurrentCaseException, TskCoreException { SleuthkitCase skCase = getCase(); Blackboard blackboard = skCase.getBlackboard(); - - Long dataSourceId = cacheKey.getParamData().getDataSourceId(); BlackboardArtifact.Type artType = cacheKey.getParamData().getArtifactType(); - - // get analysis results + List arts = new ArrayList<>(); - if (dataSourceId != null) { - arts.addAll(blackboard.getAnalysisResultsByType(artType.getTypeID(), dataSourceId)); - } else { - arts.addAll(blackboard.getAnalysisResultsByType(artType.getTypeID())); - } - - List pagedArtifacts = getPaged(arts, cacheKey); - TableData tableData = createTableData(artType, pagedArtifacts); - return new AnalysisResultTableSearchResultsDTO(artType, tableData.columnKeys, tableData.rows, cacheKey.getStartItem(), arts.size()); + String pagedWhereClause = getWhereClause(cacheKey); + arts.addAll(blackboard.getAnalysisResultsWhere(pagedWhereClause)); + blackboard.loadBlackboardAttributes(arts); + + // Get total number of results + long totalResultsCount = getTotalResultsCount(cacheKey, arts.size()); + + TableData tableData = createTableData(artType, arts); + return new AnalysisResultTableSearchResultsDTO(artType, tableData.columnKeys, tableData.rows, cacheKey.getStartItem(), totalResultsCount); } private AnalysisResultTableSearchResultsDTO fetchSetNameHitsForTable(SearchParams cacheKey) throws NoCurrentCaseException, TskCoreException { @@ -139,7 +136,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { Long dataSourceId = cacheKey.getParamData().getDataSourceId(); BlackboardArtifact.Type artType = cacheKey.getParamData().getArtifactType(); - + // Get all hash set hits List allHashHits; if (dataSourceId != null) { @@ -237,7 +234,7 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { + "Received artifact type: {0}; data source id: {1}", artType, artifactKey.getDataSourceId() == null ? "" : artifactKey.getDataSourceId())); } - SearchParams searchParams = new SearchParams<>(artifactKey, startItem, maxCount); + SearchParams searchParams = new SearchParams<>(artifactKey, startItem, maxCount); if (hardRefresh) { analysisResultCache.invalidate(searchParams); } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java index 677a61e225..4f2805a650 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultSearchParam.java @@ -18,57 +18,14 @@ */ package org.sleuthkit.autopsy.mainui.datamodel; -import java.util.Objects; import org.sleuthkit.datamodel.BlackboardArtifact; /** * Key for analysis result in order to retrieve data from DAO. */ -public class AnalysisResultSearchParam { - private final BlackboardArtifact.Type artifactType; - private final Long dataSourceId; - +public class AnalysisResultSearchParam extends BlackboardArtifactSearchParam { + public AnalysisResultSearchParam(BlackboardArtifact.Type artifactType, Long dataSourceId) { - this.artifactType = artifactType; - this.dataSourceId = dataSourceId; - } - - public BlackboardArtifact.Type getArtifactType() { - return artifactType; - } - - public Long getDataSourceId() { - return dataSourceId; - } - - @Override - public int hashCode() { - int hash = 7; - hash = 79 * hash + Objects.hashCode(this.artifactType); - hash = 79 * hash + Objects.hashCode(this.dataSourceId); - return hash; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final AnalysisResultSearchParam other = (AnalysisResultSearchParam) obj; - if (!Objects.equals(this.artifactType, other.artifactType)) { - return false; - } - if (!Objects.equals(this.dataSourceId, other.dataSourceId)) { - return false; - } - return true; - } - - + super(artifactType, dataSourceId); + } } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactDAO.java index af0bbc6945..7118ac366e 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactDAO.java @@ -16,6 +16,7 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; @@ -218,6 +219,40 @@ abstract class BlackboardArtifactDAO { return attrTypes.stream() .anyMatch(tp -> BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.DATETIME.equals(tp.getValueType())); } + + String getWhereClause(SearchParams cacheKey) { + Long dataSourceId = cacheKey.getParamData().getDataSourceId(); + BlackboardArtifact.Type artType = cacheKey.getParamData().getArtifactType(); + + String originalWhereClause = " artifacts.artifact_type_id = " + artType.getTypeID() + " "; + if (dataSourceId != null) { + originalWhereClause += " AND artifacts.data_source_obj_id = " + dataSourceId + " "; + } + + String pagedWhereClause = originalWhereClause + + " ORDER BY artifacts.obj_id ASC" + + (cacheKey.getMaxResultsCount() != null && cacheKey.getMaxResultsCount() > 0 ? " LIMIT " + cacheKey.getMaxResultsCount() : "") + + (cacheKey.getStartItem() > 0 ? " OFFSET " + cacheKey.getStartItem() : ""); + return pagedWhereClause; + } + + long getTotalResultsCount(SearchParams cacheKey, long currentPageSize) throws TskCoreException, NoCurrentCaseException { + Blackboard blackboard = getCase().getBlackboard(); + Long dataSourceId = cacheKey.getParamData().getDataSourceId(); + BlackboardArtifact.Type artType = cacheKey.getParamData().getArtifactType(); + + if ( (cacheKey.getStartItem() == 0) // offset is zero AND + && ( (cacheKey.getMaxResultsCount() != null && currentPageSize < cacheKey.getMaxResultsCount()) // number of results is less than max + || (cacheKey.getMaxResultsCount() == null)) ) { // OR max number of results was not specified + return currentPageSize; + } else { + if (dataSourceId != null) { + return blackboard.getArtifactsCount(artType.getTypeID(), dataSourceId); + } else { + return blackboard.getArtifactsCount(artType.getTypeID()); + } + } + } String getDataSourceName(Content srcContent) throws TskCoreException { Content dataSource = srcContent.getDataSource(); diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactSearchParam.java new file mode 100644 index 0000000000..2bfd9a3929 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/BlackboardArtifactSearchParam.java @@ -0,0 +1,72 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.mainui.datamodel; + +import java.util.Objects; +import org.sleuthkit.datamodel.BlackboardArtifact; + +/** + * Key for data artifact in order to retrieve data from DAO. + */ +public class BlackboardArtifactSearchParam { + private final BlackboardArtifact.Type artifactType; + private final Long dataSourceId; + + public BlackboardArtifactSearchParam(BlackboardArtifact.Type artifactType, Long dataSourceId) { + this.artifactType = artifactType; + this.dataSourceId = dataSourceId; + } + + public BlackboardArtifact.Type getArtifactType() { + return artifactType; + } + + public Long getDataSourceId() { + return dataSourceId; + } + + @Override + public int hashCode() { + int hash = 7; + hash = 67 * hash + Objects.hashCode(this.artifactType); + hash = 67 * hash + Objects.hashCode(this.dataSourceId); + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final BlackboardArtifactSearchParam other = (BlackboardArtifactSearchParam) obj; + if (!Objects.equals(this.artifactType, other.artifactType)) { + return false; + } + if (!Objects.equals(this.dataSourceId, other.dataSourceId)) { + return false; + } + return true; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java index 7ff31b270d..f5b140e79d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactDAO.java @@ -32,6 +32,7 @@ import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataArtifact; +import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** @@ -49,25 +50,24 @@ public class DataArtifactDAO extends BlackboardArtifactDAO { return instance; } - private final Cache, DataArtifactTableSearchResultsDTO> dataArtifactCache = CacheBuilder.newBuilder().maximumSize(1000).build(); + private final Cache, DataArtifactTableSearchResultsDTO> dataArtifactCache = CacheBuilder.newBuilder().maximumSize(1000).build(); - private DataArtifactTableSearchResultsDTO fetchDataArtifactsForTable(SearchParams cacheKey) throws NoCurrentCaseException, TskCoreException { - Blackboard blackboard = getCase().getBlackboard(); - - Long dataSourceId = cacheKey.getParamData().getDataSourceId(); + private DataArtifactTableSearchResultsDTO fetchDataArtifactsForTable(SearchParams cacheKey) throws NoCurrentCaseException, TskCoreException { + + SleuthkitCase skCase = getCase(); + Blackboard blackboard = skCase.getBlackboard(); BlackboardArtifact.Type artType = cacheKey.getParamData().getArtifactType(); - // get analysis results + String pagedWhereClause = getWhereClause(cacheKey); + List arts = new ArrayList<>(); - if (dataSourceId != null) { - arts.addAll(blackboard.getDataArtifacts(artType.getTypeID(), dataSourceId)); - } else { - arts.addAll(blackboard.getDataArtifacts(artType.getTypeID())); - } - - List pagedArtifacts = getPaged(arts, cacheKey); - TableData tableData = createTableData(artType, pagedArtifacts); - return new DataArtifactTableSearchResultsDTO(artType, tableData.columnKeys, tableData.rows, cacheKey.getStartItem(), arts.size()); + arts.addAll(blackboard.getDataArtifactsWhere(pagedWhereClause)); + blackboard.loadBlackboardAttributes(arts); + + long totalResultsCount = getTotalResultsCount(cacheKey, arts.size()); + + TableData tableData = createTableData(artType, arts); + return new DataArtifactTableSearchResultsDTO(artType, tableData.columnKeys, tableData.rows, cacheKey.getStartItem(), totalResultsCount); } @Override @@ -88,7 +88,7 @@ public class DataArtifactDAO extends BlackboardArtifactDAO { + "Received artifact type: {0}; data source id: {1}", artType, artifactKey.getDataSourceId() == null ? "" : artifactKey.getDataSourceId())); } - SearchParams searchParams = new SearchParams<>(artifactKey, startItem, maxCount); + SearchParams searchParams = new SearchParams<>(artifactKey, startItem, maxCount); if (hardRefresh) { this.dataArtifactCache.invalidate(searchParams); } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java index 89200ea2da..0956cf4b5c 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/DataArtifactSearchParam.java @@ -18,58 +18,14 @@ */ package org.sleuthkit.autopsy.mainui.datamodel; -import java.util.Objects; import org.sleuthkit.datamodel.BlackboardArtifact; /** * Key for data artifact in order to retrieve data from DAO. */ -public class DataArtifactSearchParam { - private final BlackboardArtifact.Type artifactType; - private final Long dataSourceId; +public class DataArtifactSearchParam extends BlackboardArtifactSearchParam { public DataArtifactSearchParam(BlackboardArtifact.Type artifactType, Long dataSourceId) { - this.artifactType = artifactType; - this.dataSourceId = dataSourceId; - } - - public BlackboardArtifact.Type getArtifactType() { - return artifactType; - } - - public Long getDataSourceId() { - return dataSourceId; - } - - @Override - public int hashCode() { - int hash = 7; - hash = 67 * hash + Objects.hashCode(this.artifactType); - hash = 67 * hash + Objects.hashCode(this.dataSourceId); - return hash; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final DataArtifactSearchParam other = (DataArtifactSearchParam) obj; - if (!Objects.equals(this.artifactType, other.artifactType)) { - return false; - } - if (!Objects.equals(this.dataSourceId, other.dataSourceId)) { - return false; - } - return true; - } - - - + super (artifactType, dataSourceId); + } } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java index 64097081a6..86ae8909f2 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java @@ -72,6 +72,8 @@ public class TableSearchTest extends NbTestCase { private static final String ARTIFACT_CONFIGURATION = "Test configuration"; private static final String ARTIFACT_JUSTIFICATION = "Test justification"; private static final Score ARTIFACT_SCORE = Score.SCORE_LIKELY_NOTABLE; + private static final long ARTIFACT_COUNT_WEB_BOOKMARK = 125; + private static final long ARTIFACT_COUNT_YARA = 150; // Values for the hash set hit tests private static final String HASH_SET_1 = "Hash Set 1"; @@ -226,6 +228,13 @@ public class TableSearchTest extends NbTestCase { customDataArtifactSourceFile = fileA3; customDataArtifactLinkedFile = fileA2; + // Add a lot of web bookmark data artifacts + for (int i = 0;i < ARTIFACT_COUNT_WEB_BOOKMARK;i++) { + attrs.clear(); + attrs.add(new BlackboardAttribute(BlackboardAttribute.Type.TSK_COMMENT, MODULE_NAME, Integer.toString(i))); + fileA1.newDataArtifact(BlackboardArtifact.Type.TSK_WEB_BOOKMARK, attrs); + } + // Add analysis results // Data source 1: Encryption detected (2), custom type // Data source 2: Encryption detected @@ -250,6 +259,13 @@ public class TableSearchTest extends NbTestCase { customAnalysisResult = customDataArtifact.newAnalysisResult(customAnalysisResultType, ARTIFACT_SCORE, ARTIFACT_CONCLUSION, ARTIFACT_CONFIGURATION, ARTIFACT_JUSTIFICATION, attrs).getAnalysisResult(); customAnalysisResultSource = customDataArtifact; + // Add a lot of YARA hit analysis results + for (int i = 0;i < ARTIFACT_COUNT_YARA;i++) { + attrs.clear(); + attrs.add(new BlackboardAttribute(BlackboardAttribute.Type.TSK_COMMENT, MODULE_NAME, Integer.toString(i))); + fileA1.newAnalysisResult(BlackboardArtifact.Type.TSK_YARA_HIT, Score.SCORE_NOTABLE, "conclusion", "configuration", "justification", attrs); + } + // Add hash hits attrs.clear(); attrs.add(new BlackboardAttribute(BlackboardAttribute.Type.TSK_SET_NAME, MODULE_NAME, HASH_SET_1)); @@ -373,6 +389,41 @@ public class TableSearchTest extends NbTestCase { assertTrue(dataArtifactRowDTO.getCellValues().contains(ARTIFACT_INT)); assertTrue(dataArtifactRowDTO.getCellValues().contains(ARTIFACT_DOUBLE)); + // Test paging + Long pageSize = new Long(100); + assertTrue(ARTIFACT_COUNT_WEB_BOOKMARK > pageSize); + + // Get the first page + param = new DataArtifactSearchParam(BlackboardArtifact.Type.TSK_WEB_BOOKMARK, null); + results = dataArtifactDAO.getDataArtifactsForTable(param, 0, pageSize, false); + assertEquals(ARTIFACT_COUNT_WEB_BOOKMARK, results.getTotalResultsCount()); + assertEquals(pageSize.longValue(), results.getItems().size()); + + // Save all artifact IDs from the first page + Set firstPageObjIds = new HashSet<>(); + for (RowDTO row : results.getItems()) { + assertTrue(row instanceof DataArtifactRowDTO); + DataArtifactRowDTO dataRow = (DataArtifactRowDTO) row; + assertTrue(dataRow.getDataArtifact() != null); + firstPageObjIds.add(dataRow.getDataArtifact().getId()); + } + assertEquals(pageSize.longValue(), firstPageObjIds.size()); + + // Get the second page + param = new DataArtifactSearchParam(BlackboardArtifact.Type.TSK_WEB_BOOKMARK, null); + results = dataArtifactDAO.getDataArtifactsForTable(param, pageSize, pageSize, false); + assertEquals(ARTIFACT_COUNT_WEB_BOOKMARK, results.getTotalResultsCount()); + assertEquals(ARTIFACT_COUNT_WEB_BOOKMARK - pageSize, results.getItems().size()); + + // Make sure no artifacts from the second page appeared on the first + for (RowDTO row : results.getItems()) { + assertTrue(row instanceof DataArtifactRowDTO); + DataArtifactRowDTO dataRow = (DataArtifactRowDTO) row; + assertTrue(dataRow.getDataArtifact() != null); + assertFalse("Data artifact ID: " + dataRow.getDataArtifact().getId() + " appeared on both page 1 and page 2", + firstPageObjIds.contains(dataRow.getDataArtifact().getId())); + } + } catch (ExecutionException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex.getMessage()); @@ -528,6 +579,41 @@ public class TableSearchTest extends NbTestCase { assertTrue(analysisResultRowDTO.getCellValues().contains(ARTIFACT_CONFIGURATION)); assertTrue(analysisResultRowDTO.getCellValues().contains(ARTIFACT_CONCLUSION)); + // Test paging + Long pageSize = new Long(100); + assertTrue(ARTIFACT_COUNT_YARA > pageSize); + + // Get the first page + param = new AnalysisResultSearchParam(BlackboardArtifact.Type.TSK_YARA_HIT, null); + results = analysisResultDAO.getAnalysisResultsForTable(param, 0, pageSize, false); + assertEquals(ARTIFACT_COUNT_YARA, results.getTotalResultsCount()); + assertEquals(pageSize.longValue(), results.getItems().size()); + + // Save all artifact IDs from the first page + Set firstPageObjIds = new HashSet<>(); + for (RowDTO row : results.getItems()) { + assertTrue(row instanceof AnalysisResultRowDTO); + AnalysisResultRowDTO analysisRow = (AnalysisResultRowDTO) row; + assertTrue(analysisRow.getAnalysisResult() != null); + firstPageObjIds.add(analysisRow.getAnalysisResult().getId()); + } + assertEquals(pageSize.longValue(), firstPageObjIds.size()); + + // Get the second page + param = new AnalysisResultSearchParam(BlackboardArtifact.Type.TSK_YARA_HIT, null); + results = analysisResultDAO.getAnalysisResultsForTable(param, pageSize, pageSize, false); + assertEquals(ARTIFACT_COUNT_YARA, results.getTotalResultsCount()); + assertEquals(ARTIFACT_COUNT_YARA - pageSize, results.getItems().size()); + + // Make sure no artifacts from the second page appeared on the first + for (RowDTO row : results.getItems()) { + assertTrue(row instanceof AnalysisResultRowDTO); + AnalysisResultRowDTO analysisRow = (AnalysisResultRowDTO) row; + assertTrue(analysisRow.getAnalysisResult() != null); + assertFalse("Analysis result ID: " + analysisRow.getAnalysisResult().getId() + " appeared on both page 1 and page 2", + firstPageObjIds.contains(analysisRow.getAnalysisResult().getId())); + } + } catch (ExecutionException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex.getMessage()); From 5a44fe0aaa95cebbe6b8ebdcec1c06cb5ae0d74b Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 28 Oct 2021 15:09:17 -0400 Subject: [PATCH 05/12] First cut at functional tests --- .../autopsy/mainui/datamodel/MainDAO.java | 5 ++ .../mainui/datamodel/TableSearchTest.java | 80 ++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java index 5849ca88d5..f17c5fe52d 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/MainDAO.java @@ -37,6 +37,7 @@ public class MainDAO { private final DataArtifactDAO dataArtifactDAO = DataArtifactDAO.getInstance(); private final AnalysisResultDAO analysisResultDAO = AnalysisResultDAO.getInstance(); private final ViewsDAO viewsDAO = ViewsDAO.getInstance(); + private final TagsDAO tagsDAO = TagsDAO.getInstance(); public DataArtifactDAO getDataArtifactsDAO() { return dataArtifactDAO; @@ -49,4 +50,8 @@ public class MainDAO { public ViewsDAO getViewsDAO() { return viewsDAO; } + + public TagsDAO getTagsDAO() { + return tagsDAO; + } } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java index 64097081a6..297b194b18 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java @@ -32,6 +32,7 @@ import org.netbeans.junit.NbModuleSuite; import org.netbeans.junit.NbTestCase; import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.testutils.CaseUtils; import org.sleuthkit.autopsy.testutils.TestUtilsException; import org.sleuthkit.datamodel.AbstractFile; @@ -45,6 +46,7 @@ import org.sleuthkit.datamodel.DataArtifact; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskCoreException; @@ -85,6 +87,7 @@ public class TableSearchTest extends NbTestCase { private static final String KEYWORD_PREVIEW = "There is a bomb."; // Extension and MIME type test + private static AbstractFile customFile; private static final String CUSTOM_MIME_TYPE = "fake/type"; private static final String CUSTOM_MIME_TYPE_FILE_NAME = "test.fake"; private static final String CUSTOM_EXTENSION = "fake"; @@ -98,6 +101,7 @@ public class TableSearchTest extends NbTestCase { Case openCase = null; // The case for testing SleuthkitCase db = null; // The case database Blackboard blackboard = null; // The blackboard + TagsManager tagsManager = null;// Tags manager DataSource dataSource1 = null; // A local files data source DataSource dataSource2 = null; // A local files data source @@ -123,6 +127,10 @@ public class TableSearchTest extends NbTestCase { // Keyword hits test AnalysisResult keywordHitAnalysisResult = null; // A keyword hit Content keywordHitSource = null; // The source of the keyword hit above + + // Tags test + TagName knownTag1 = null; + TagName tag2 = null; public static Test suite() { NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(TableSearchTest.class). @@ -148,6 +156,7 @@ public class TableSearchTest extends NbTestCase { mimeSearchTest(); extensionSearchTest(); sizeSearchTest(); + tagsTest(); } /** @@ -159,6 +168,7 @@ public class TableSearchTest extends NbTestCase { openCase = CaseUtils.createAsCurrentCase("testTableSearchCase"); db = openCase.getSleuthkitCase(); blackboard = db.getBlackboard(); + tagsManager = openCase.getServices().getTagsManager(); // Add two logical files data sources SleuthkitCase.CaseDbTransaction trans = db.beginTransaction(); @@ -189,7 +199,7 @@ public class TableSearchTest extends NbTestCase { fileB1.setMIMEType("text/plain"); fileB1.save(); - AbstractFile customFile = db.addLocalFile(CUSTOM_MIME_TYPE_FILE_NAME, "", 67000000, 0, 0, 0, 0, true, TskData.EncodingType.NONE, folderB1); + customFile = db.addLocalFile(CUSTOM_MIME_TYPE_FILE_NAME, "", 67000000, 0, 0, 0, 0, true, TskData.EncodingType.NONE, folderB1); customFile.setMIMEType(CUSTOM_MIME_TYPE); customFile.save(); @@ -306,7 +316,25 @@ public class TableSearchTest extends NbTestCase { null, KEYWORD_SET_1, null, attrs).getAnalysisResult(); keywordHitSource = hashHitAnalysisResult; - } catch (TestUtilsException | TskCoreException | BlackboardException ex) { + // Add tags ---- + knownTag1 = tagsManager.addTagName("Tag 1", "Descrition", TagName.HTML_COLOR.RED, TskData.FileKnown.KNOWN); + tag2 = tagsManager.addTagName("Tag 2", "Descrition"); + + // Tag the custom artifacts in data source 1 + openCase.getServices().getTagsManager().addBlackboardArtifactTag(customDataArtifact, knownTag1, "Comment"); + openCase.getServices().getTagsManager().addBlackboardArtifactTag(customAnalysisResult, tag2, "Comment 2"); + + // Tag file in data source 1 + openCase.getServices().getTagsManager().addContentTag(fileA2, tag2); + openCase.getServices().getTagsManager().addContentTag(fileA3, tag2); + + // Tag file in data source 2 + openCase.getServices().getTagsManager().addContentTag(fileB1, tag2); + + // Tag the custom file in data source 2 + openCase.getServices().getTagsManager().addContentTag(customFile, knownTag1); + + } catch (TestUtilsException | TskCoreException | BlackboardException | TagsManager.TagNameAlreadyExistsException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex.getMessage()); } @@ -476,6 +504,52 @@ public class TableSearchTest extends NbTestCase { } } + public void tagsTest() { + // Quick test that everything is initialized + assertTrue(db != null); + + try { + TagsDAO tagsDAO = MainDAO.getInstance().getTagsDAO(); + + // Get "Tag1" file tags from data source 1 + TagsSearchParams param = new TagsSearchParams(knownTag1, TagsSearchParams.TagType.FILE, dataSource1.getId()); + SearchResultsDTO results = tagsDAO.getTags(param, 0, null, false); + assertEquals(0, results.getTotalResultsCount()); + assertEquals(0, results.getItems().size()); + + // Get "Tag2" file tags from data source 1 + param = new TagsSearchParams(tag2, TagsSearchParams.TagType.FILE, dataSource1.getId()); + results = tagsDAO.getTags(param, 0, null, false); + assertEquals(2, results.getTotalResultsCount()); + assertEquals(2, results.getItems().size()); + + // Get "Tag2" file tags from all data sources + param = new TagsSearchParams(tag2, TagsSearchParams.TagType.FILE, null); + results = tagsDAO.getTags(param, 0, null, false); + assertEquals(3, results.getTotalResultsCount()); + assertEquals(3, results.getItems().size()); + + // Get "Tag1" result tags from data source 1 + param = new TagsSearchParams(knownTag1, TagsSearchParams.TagType.RESULT, dataSource1.getId()); + results = tagsDAO.getTags(param, 0, null, false); + assertEquals(1, results.getTotalResultsCount()); + assertEquals(1, results.getItems().size()); + + // Get "Tag1" result tags from data source 2 + param = new TagsSearchParams(knownTag1, TagsSearchParams.TagType.RESULT, dataSource2.getId()); + results = tagsDAO.getTags(param, 0, null, false); + assertEquals(0, results.getTotalResultsCount()); + assertEquals(0, results.getItems().size()); + + // Test custom tags + + + } catch (ExecutionException ex) { + Exceptions.printStackTrace(ex); + Assert.fail(ex.getMessage()); + } + } + public void analysisResultSearchTest() { // Quick test that everything is initialized assertTrue(db != null); @@ -721,5 +795,7 @@ public class TableSearchTest extends NbTestCase { } openCase = null; db = null; + blackboard = null; + tagsManager = null; } } From 39332f78db5a0212107324c679c2629fa7aab2e2 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 28 Oct 2021 16:13:43 -0400 Subject: [PATCH 06/12] Functional tests --- .../mainui/datamodel/TableSearchTest.java | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java index 297b194b18..8fd5209e3a 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java @@ -94,6 +94,15 @@ public class TableSearchTest extends NbTestCase { private static final Set CUSTOM_EXTENSIONS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("." + CUSTOM_EXTENSION))); //NON-NLS private static final Set EMPTY_RESULT_SET_EXTENSIONS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(".blah", ".blah2", ".crazy"))); //NON-NLS + // Tag test + private static final String TAG_COMMENT = "Tag comment"; + private static final String TAG_DESCRIPTION = "Tag description"; + private static final String MD5_COLUMN = "MD5 Hash"; + private static final String FILE_PATH_COLUMN = "File Path"; + private static final String MODIFIED_TIME_COLUMN = "Modified Time"; + private static final String SOURCE_NAME_COLUMN = "Source Name"; + private static final String SOURCE_FILE_PATH_COLUMN = "Source File Path"; + ///////////////////////////////////////////////// // Data to be used across the test methods. // These are initialized in setUpCaseDatabase(). @@ -317,11 +326,11 @@ public class TableSearchTest extends NbTestCase { keywordHitSource = hashHitAnalysisResult; // Add tags ---- - knownTag1 = tagsManager.addTagName("Tag 1", "Descrition", TagName.HTML_COLOR.RED, TskData.FileKnown.KNOWN); + knownTag1 = tagsManager.addTagName("Tag 1", TAG_DESCRIPTION, TagName.HTML_COLOR.RED, TskData.FileKnown.KNOWN); tag2 = tagsManager.addTagName("Tag 2", "Descrition"); // Tag the custom artifacts in data source 1 - openCase.getServices().getTagsManager().addBlackboardArtifactTag(customDataArtifact, knownTag1, "Comment"); + openCase.getServices().getTagsManager().addBlackboardArtifactTag(customDataArtifact, knownTag1, TAG_COMMENT); openCase.getServices().getTagsManager().addBlackboardArtifactTag(customAnalysisResult, tag2, "Comment 2"); // Tag file in data source 1 @@ -528,6 +537,28 @@ public class TableSearchTest extends NbTestCase { results = tagsDAO.getTags(param, 0, null, false); assertEquals(3, results.getTotalResultsCount()); assertEquals(3, results.getItems().size()); + + // Check that a few of the expected file tag column names are present + List columnDisplayNames = results.getColumns().stream().map(p -> p.getDisplayName()).collect(Collectors.toList()); + assertTrue(columnDisplayNames.contains(MD5_COLUMN)); + assertTrue(columnDisplayNames.contains(FILE_PATH_COLUMN)); + assertTrue(columnDisplayNames.contains(MODIFIED_TIME_COLUMN)); + + // Check that the result tag columns are not present + assertFalse(columnDisplayNames.contains(SOURCE_NAME_COLUMN)); + assertFalse(columnDisplayNames.contains(SOURCE_FILE_PATH_COLUMN)); + + // Get "Tag1" result tags from data source 2 + param = new TagsSearchParams(knownTag1, TagsSearchParams.TagType.RESULT, dataSource2.getId()); + results = tagsDAO.getTags(param, 0, null, false); + assertEquals(0, results.getTotalResultsCount()); + assertEquals(0, results.getItems().size()); + + // Get "Tag2" result tags from data source 1 + param = new TagsSearchParams(tag2, TagsSearchParams.TagType.RESULT, dataSource1.getId()); + results = tagsDAO.getTags(param, 0, null, false); + assertEquals(1, results.getTotalResultsCount()); + assertEquals(1, results.getItems().size()); // Get "Tag1" result tags from data source 1 param = new TagsSearchParams(knownTag1, TagsSearchParams.TagType.RESULT, dataSource1.getId()); @@ -535,14 +566,13 @@ public class TableSearchTest extends NbTestCase { assertEquals(1, results.getTotalResultsCount()); assertEquals(1, results.getItems().size()); - // Get "Tag1" result tags from data source 2 - param = new TagsSearchParams(knownTag1, TagsSearchParams.TagType.RESULT, dataSource2.getId()); - results = tagsDAO.getTags(param, 0, null, false); - assertEquals(0, results.getTotalResultsCount()); - assertEquals(0, results.getItems().size()); - - // Test custom tags + // Get the row + RowDTO rowDTO = results.getItems().get(0); + assertTrue(rowDTO instanceof BaseRowDTO); + BaseRowDTO tagResultRowDTO = (BaseRowDTO) rowDTO; + // Check that some of the expected result tag column values are present + assertTrue(tagResultRowDTO.getCellValues().contains(TAG_COMMENT)); } catch (ExecutionException ex) { Exceptions.printStackTrace(ex); From 05fdf573e92b59f13d0499af58c12366d5b54406 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 28 Oct 2021 16:24:05 -0400 Subject: [PATCH 07/12] More functional tests --- .../mainui/datamodel/TableSearchTest.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java index 8fd5209e3a..d4cf05dbae 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/mainui/datamodel/TableSearchTest.java @@ -538,6 +538,20 @@ public class TableSearchTest extends NbTestCase { assertEquals(3, results.getTotalResultsCount()); assertEquals(3, results.getItems().size()); + // Get "Tag1" file tags from data source 2 + param = new TagsSearchParams(knownTag1, TagsSearchParams.TagType.FILE, dataSource2.getId()); + results = tagsDAO.getTags(param, 0, null, false); + assertEquals(1, results.getTotalResultsCount()); + assertEquals(1, results.getItems().size()); + + // Get the row + RowDTO rowDTO = results.getItems().get(0); + assertTrue(rowDTO instanceof BaseRowDTO); + BaseRowDTO tagResultRowDTO = (BaseRowDTO) rowDTO; + + // Check that the file tag is for the custom file + assertTrue(tagResultRowDTO.getCellValues().contains(customFile.getName())); + // Check that a few of the expected file tag column names are present List columnDisplayNames = results.getColumns().stream().map(p -> p.getDisplayName()).collect(Collectors.toList()); assertTrue(columnDisplayNames.contains(MD5_COLUMN)); @@ -567,9 +581,9 @@ public class TableSearchTest extends NbTestCase { assertEquals(1, results.getItems().size()); // Get the row - RowDTO rowDTO = results.getItems().get(0); + rowDTO = results.getItems().get(0); assertTrue(rowDTO instanceof BaseRowDTO); - BaseRowDTO tagResultRowDTO = (BaseRowDTO) rowDTO; + tagResultRowDTO = (BaseRowDTO) rowDTO; // Check that some of the expected result tag column values are present assertTrue(tagResultRowDTO.getCellValues().contains(TAG_COMMENT)); From 4f018b328130555015cccb703d1175582585e537 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 29 Oct 2021 10:39:34 -0400 Subject: [PATCH 08/12] Cleaning things up --- .../autopsy/mainui/datamodel/TagsDAO.java | 77 ++++--------------- 1 file changed, 13 insertions(+), 64 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java index d71f199191..5c6059fa5d 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java @@ -32,18 +32,15 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.TimeZoneUtils; -import org.sleuthkit.autopsy.datamodel.FileTypeExtensions; -import org.sleuthkit.autopsy.mainui.datamodel.FileRowDTO.ExtensionMediaType; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifactTag; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; -import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; /** - * Provides information to populate the results viewer for data in the views + * Provides information to populate the results viewer for data in the tags * section. */ @Messages({"TagsDAO.fileColumns.nameColLbl=Name", @@ -66,7 +63,7 @@ import org.sleuthkit.datamodel.TskCoreException; "TagsDAO.tagColumns.userNameColLbl=User Name"}) public class TagsDAO { - private static final int CACHE_SIZE = 15; // rule of thumb: 5 entries times number of cached SearchParams sub-types + private static final int CACHE_SIZE = 5; // rule of thumb: 5 entries times number of cached SearchParams sub-types private static final long CACHE_DURATION = 2; private static final TimeUnit CACHE_DURATION_UNITS = TimeUnit.MINUTES; private final Cache, SearchResultsDTO> searchParamsCache = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build(); @@ -75,12 +72,10 @@ public class TagsDAO { private static final String FILE_TAG_TYPE_ID = "FILE_TAG"; private static final String RESULT_TAG_TYPE_ID = "RESULT_TAG"; - private static final String FILE_TAG_DISPLAY_NAME = "File Tag"; - private static final String RESULT_TAG_DISPLAY_NAME = "Result Tag"; private static final List FILE_TAG_COLUMNS = Arrays.asList( getFileColumnKey(Bundle.TagsDAO_fileColumns_nameColLbl()), - getFileColumnKey(Bundle.TagsDAO_fileColumns_originalName()), // ELODO handle translation + getFileColumnKey(Bundle.TagsDAO_fileColumns_originalName()), // GVDTODO handle translation getFileColumnKey(Bundle.TagsDAO_fileColumns_filePathColLbl()), getFileColumnKey(Bundle.TagsDAO_fileColumns_commentColLbl()), getFileColumnKey(Bundle.TagsDAO_fileColumns_modifiedTimeColLbl()), @@ -112,45 +107,14 @@ public class TagsDAO { private static ColumnKey getFileColumnKey(String name) { return new ColumnKey(name, name, Bundle.TagsDAO_fileColumns_noDescription()); } - - private SleuthkitCase getCase() throws NoCurrentCaseException { - return Case.getCurrentCaseThrows().getSleuthkitCase(); - } - - static ExtensionMediaType getExtensionMediaType(String ext) { - if (StringUtils.isBlank(ext)) { - return ExtensionMediaType.UNCATEGORIZED; - } else { - ext = "." + ext; - } - if (FileTypeExtensions.getImageExtensions().contains(ext)) { - return ExtensionMediaType.IMAGE; - } else if (FileTypeExtensions.getVideoExtensions().contains(ext)) { - return ExtensionMediaType.VIDEO; - } else if (FileTypeExtensions.getAudioExtensions().contains(ext)) { - return ExtensionMediaType.AUDIO; - } else if (FileTypeExtensions.getDocumentExtensions().contains(ext)) { - return ExtensionMediaType.DOC; - } else if (FileTypeExtensions.getExecutableExtensions().contains(ext)) { - return ExtensionMediaType.EXECUTABLE; - } else if (FileTypeExtensions.getTextExtensions().contains(ext)) { - return ExtensionMediaType.TEXT; - } else if (FileTypeExtensions.getWebExtensions().contains(ext)) { - return ExtensionMediaType.WEB; - } else if (FileTypeExtensions.getPDFExtensions().contains(ext)) { - return ExtensionMediaType.PDF; - } else if (FileTypeExtensions.getArchiveExtensions().contains(ext)) { - return ExtensionMediaType.ARCHIVE; - } else { - return ExtensionMediaType.UNCATEGORIZED; - } - } public SearchResultsDTO getTags(TagsSearchParams key, long startItem, Long maxCount, boolean hardRefresh) throws ExecutionException, IllegalArgumentException { if (key.getTagName() == null) { - throw new IllegalArgumentException("Must have non-null filter"); + throw new IllegalArgumentException("Must have non-null tag name"); } else if (key.getDataSourceId() != null && key.getDataSourceId() <= 0) { throw new IllegalArgumentException("Data source id must be greater than 0 or null"); + } else if (key.getTagType() == null) { + throw new IllegalArgumentException("Must have non-null tag type"); } SearchParams searchParams = new SearchParams<>(key, startItem, maxCount); @@ -161,34 +125,19 @@ public class TagsDAO { return searchParamsCache.get(searchParams, () -> fetchTagsDTOs(key.getTagName(), key.getTagType(), key.getDataSourceId(), startItem, maxCount)); } - @NbBundle.Messages({"FileTag.name.text=File Tag"}) + @NbBundle.Messages({"FileTag.name.text=File Tag", + "ResultTag.name.text=Result Tag"}) private SearchResultsDTO fetchTagsDTOs(TagName tagName, TagsSearchParams.TagType type, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { - if (null == type) { - // ELTODO throw? - return null; - } - switch (type) { case FILE: return fetchFileTags(tagName, dataSourceId, startItem, maxResultCount); case RESULT: return fetchResultTags(tagName, dataSourceId, startItem, maxResultCount); default: - // ELTODO throw? - return null; + throw new IllegalArgumentException("Unsupported tag type"); } } - /* GET RESULT TAGS - * BlackboardArtifactTagNodeFactory.createKeys(List tags) - * BlackboardArtifactTagNode.createSheet() - */ - - /* GET FILE TAGS - * ContentTagNodeFactory.createKeys(List tags) - * ContentTagNode.createSheet() - */ - private SearchResultsDTO fetchResultTags(TagName tagName, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { // ELTODO startItem, maxResultCount @@ -227,7 +176,7 @@ public class TagsDAO { List cellValues = Arrays.asList( name, - null, // ELTODO translation column + null, // GVDTODO translation column contentPath, tag.getArtifact().getDisplayName(), tag.getComment(), @@ -239,7 +188,7 @@ public class TagsDAO { tag.getId())); } - return new BaseSearchResultsDTO(RESULT_TAG_TYPE_ID, RESULT_TAG_DISPLAY_NAME, RESULT_TAG_COLUMNS, fileRows, startItem, fileRows.size()); + return new BaseSearchResultsDTO(RESULT_TAG_TYPE_ID, Bundle.ResultTag_name_text(), RESULT_TAG_COLUMNS, fileRows, 0, fileRows.size()); } private SearchResultsDTO fetchFileTags(TagName tagName, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { @@ -269,7 +218,7 @@ public class TagsDAO { List cellValues = Arrays.asList( content.getName(), - null, // ELTODO translation column + null, // GVDTODO translation column contentPath, tag.getComment(), file != null ? TimeZoneUtils.getFormattedTime(file.getMtime()) : "", @@ -286,6 +235,6 @@ public class TagsDAO { file.getId())); } - return new BaseSearchResultsDTO(FILE_TAG_TYPE_ID, FILE_TAG_DISPLAY_NAME, FILE_TAG_COLUMNS, fileRows, startItem, fileRows.size()); + return new BaseSearchResultsDTO(FILE_TAG_TYPE_ID, Bundle.FileTag_name_text(), FILE_TAG_COLUMNS, fileRows, 0, fileRows.size()); } } From ee81e21ac9420a457d964e8325c3926f76ef1752 Mon Sep 17 00:00:00 2001 From: apriestman Date: Fri, 29 Oct 2021 11:07:20 -0400 Subject: [PATCH 09/12] Slightly optimize hash hits --- .../mainui/datamodel/AnalysisResultDAO.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java index c044fa8587..16e1976870 100644 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/AnalysisResultDAO.java @@ -136,27 +136,29 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { Long dataSourceId = cacheKey.getParamData().getDataSourceId(); BlackboardArtifact.Type artType = cacheKey.getParamData().getArtifactType(); - - // Get all hash set hits - List allHashHits; - if (dataSourceId != null) { - allHashHits = blackboard.getAnalysisResultsByType(artType.getTypeID(), dataSourceId); - } else { - allHashHits = blackboard.getAnalysisResultsByType(artType.getTypeID()); - } + // We currently can't make a query on the set name field because need to use a prepared statement + String originalWhereClause = " artifacts.artifact_type_id = " + artType.getTypeID() + " "; + if (dataSourceId != null) { + originalWhereClause += " AND artifacts.data_source_obj_id = " + dataSourceId + " "; + } + + List allHashHits = new ArrayList<>(); + allHashHits.addAll(blackboard.getAnalysisResultsWhere(originalWhereClause)); + blackboard.loadBlackboardAttributes(allHashHits); + // Filter for the selected set - List arts = new ArrayList<>(); - for (AnalysisResult art : allHashHits) { + List hashHits = new ArrayList<>(); + for (BlackboardArtifact art : allHashHits) { BlackboardAttribute setNameAttr = art.getAttribute(BlackboardAttribute.Type.TSK_SET_NAME); if ((setNameAttr != null) && cacheKey.getParamData().getSetName().equals(setNameAttr.getValueString())) { - arts.add(art); + hashHits.add(art); } } - List pagedArtifacts = getPaged(arts, cacheKey); + List pagedArtifacts = getPaged(hashHits, cacheKey); TableData tableData = createTableData(artType, pagedArtifacts); - return new AnalysisResultTableSearchResultsDTO(artType, tableData.columnKeys, tableData.rows, cacheKey.getStartItem(), arts.size()); + return new AnalysisResultTableSearchResultsDTO(artType, tableData.columnKeys, tableData.rows, cacheKey.getStartItem(), hashHits.size()); } @Override @@ -261,6 +263,8 @@ public class AnalysisResultDAO extends BlackboardArtifactDAO { return hashHitCache.get(searchParams, () -> fetchSetNameHitsForTable(searchParams)); } + // TODO - JIRA-8117 + // This needs to use more than just the set name public AnalysisResultTableSearchResultsDTO getKeywordHitsForTable(KeywordHitSearchParam artifactKey, long startItem, Long maxCount, boolean hardRefresh) throws ExecutionException, IllegalArgumentException { if (artifactKey.getDataSourceId() != null && artifactKey.getDataSourceId() < 0) { throw new IllegalArgumentException(MessageFormat.format("Illegal data. " From fd99b07e307548af73a576fb606fbee3fe53b096 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 29 Oct 2021 17:33:50 -0400 Subject: [PATCH 10/12] Added paging --- .../autopsy/mainui/datamodel/TagsDAO.java | 150 +++++++++++++----- 1 file changed, 114 insertions(+), 36 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java index 5c6059fa5d..5dcf62a35a 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java @@ -20,11 +20,17 @@ package org.sleuthkit.autopsy.mainui.datamodel; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import java.beans.PropertyChangeEvent; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; +import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; @@ -32,16 +38,19 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.TimeZoneUtils; +import org.sleuthkit.autopsy.mainui.nodes.DAOFetcher; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifactTag; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; +import org.sleuthkit.datamodel.Tag; import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; /** - * Provides information to populate the results viewer for data in the tags - * section. + * Provides information to populate the results viewer for data in the allTags + section. */ @Messages({"TagsDAO.fileColumns.nameColLbl=Name", "TagsDAO.fileColumns.originalName=Original Name", @@ -122,28 +131,49 @@ public class TagsDAO { this.searchParamsCache.invalidate(searchParams); } - return searchParamsCache.get(searchParams, () -> fetchTagsDTOs(key.getTagName(), key.getTagType(), key.getDataSourceId(), startItem, maxCount)); - } + return searchParamsCache.get(searchParams, () -> fetchTagsDTOs(searchParams)); + } @NbBundle.Messages({"FileTag.name.text=File Tag", "ResultTag.name.text=Result Tag"}) - private SearchResultsDTO fetchTagsDTOs(TagName tagName, TagsSearchParams.TagType type, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { - switch (type) { + private SearchResultsDTO fetchTagsDTOs(SearchParams cacheKey) throws NoCurrentCaseException, TskCoreException { + switch (cacheKey.getParamData().getTagType()) { case FILE: - return fetchFileTags(tagName, dataSourceId, startItem, maxResultCount); + return fetchFileTags(cacheKey); case RESULT: - return fetchResultTags(tagName, dataSourceId, startItem, maxResultCount); + return fetchResultTags(cacheKey); default: throw new IllegalArgumentException("Unsupported tag type"); } } - private SearchResultsDTO fetchResultTags(TagName tagName, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { + /** + * Returns a list of paged tag results. + * + * @param tags The tag results. + * @param searchParams The search parameters including the paging. + * + * @return The list of paged tag results. + */ + List getPaged(List tags, SearchParams searchParams) { + Stream pagedTagsStream = tags.stream() + .sorted(Comparator.comparing((tag) -> tag.getId())) + .skip(searchParams.getStartItem()); - // ELTODO startItem, maxResultCount + if (searchParams.getMaxResultsCount() != null) { + pagedTagsStream = pagedTagsStream.limit(searchParams.getMaxResultsCount()); + } + + return pagedTagsStream.collect(Collectors.toList()); + } + + private SearchResultsDTO fetchResultTags(SearchParams cacheKey) throws NoCurrentCaseException, TskCoreException { + + Long dataSourceId = cacheKey.getParamData().getDataSourceId(); + TagName tagName = cacheKey.getParamData().getTagName(); - List tags = new ArrayList<>(); - // Use the blackboard artifact tags bearing the specified tag name as the tags. + // get all tag results + List allTags = new ArrayList<>(); List artifactTags = (dataSourceId != null && dataSourceId > 0) ? Case.getCurrentCaseThrows().getServices().getTagsManager().getBlackboardArtifactTagsByTagName(tagName, dataSourceId) : Case.getCurrentCaseThrows().getServices().getTagsManager().getBlackboardArtifactTagsByTagName(tagName); @@ -151,51 +181,57 @@ public class TagsDAO { String userName = System.getProperty(USER_NAME_PROPERTY); for (BlackboardArtifactTag tag : artifactTags) { if (userName.equals(tag.getUserName())) { - tags.add(tag); + allTags.add(tag); } } } else { - tags.addAll(artifactTags); + allTags.addAll(artifactTags); } + + // get current page of tag results + List pagedTags = getPaged(allTags, cacheKey); List fileRows = new ArrayList<>(); - for (BlackboardArtifactTag tag : tags) { - String name = tag.getContent().getName(); // As a backup. + for (Tag tag : pagedTags) { + BlackboardArtifactTag blackboardTag = (BlackboardArtifactTag) tag; + + String name = blackboardTag.getContent().getName(); // As a backup. try { - name = tag.getArtifact().getShortDescription(); + name = blackboardTag.getArtifact().getShortDescription(); } catch (TskCoreException ignore) { // it's a WARNING, skip } String contentPath; try { - contentPath = tag.getContent().getUniquePath(); + contentPath = blackboardTag.getContent().getUniquePath(); } catch (TskCoreException ex) { contentPath = NbBundle.getMessage(this.getClass(), "BlackboardArtifactTagNode.createSheet.unavail.text"); } - List cellValues = Arrays.asList( - name, + List cellValues = Arrays.asList(name, null, // GVDTODO translation column contentPath, - tag.getArtifact().getDisplayName(), - tag.getComment(), - tag.getUserName()); + blackboardTag.getArtifact().getDisplayName(), + blackboardTag.getComment(), + blackboardTag.getUserName()); fileRows.add(new BaseRowDTO( cellValues, RESULT_TAG_TYPE_ID, - tag.getId())); + blackboardTag.getId())); } - return new BaseSearchResultsDTO(RESULT_TAG_TYPE_ID, Bundle.ResultTag_name_text(), RESULT_TAG_COLUMNS, fileRows, 0, fileRows.size()); + return new BaseSearchResultsDTO(RESULT_TAG_TYPE_ID, Bundle.ResultTag_name_text(), RESULT_TAG_COLUMNS, fileRows, 0, allTags.size()); } - private SearchResultsDTO fetchFileTags(TagName tagName, Long dataSourceId, long startItem, Long maxResultCount) throws NoCurrentCaseException, TskCoreException { + private SearchResultsDTO fetchFileTags(SearchParams cacheKey) throws NoCurrentCaseException, TskCoreException { - // ELTODO startItem, maxResultCount + Long dataSourceId = cacheKey.getParamData().getDataSourceId(); + TagName tagName = cacheKey.getParamData().getTagName(); - List tags = new ArrayList<>(); + // get all tag results + List allTags = new ArrayList<>(); List contentTags = (dataSourceId != null && dataSourceId > 0) ? Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByTagName(tagName, dataSourceId) : Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByTagName(tagName); @@ -203,16 +239,20 @@ public class TagsDAO { String userName = System.getProperty(USER_NAME_PROPERTY); for (ContentTag tag : contentTags) { if (userName.equals(tag.getUserName())) { - tags.add(tag); + allTags.add(tag); } } } else { - tags.addAll(contentTags); + allTags.addAll(contentTags); } - + + // get current page of tag results + List pagedTags = getPaged(allTags, cacheKey); + List fileRows = new ArrayList<>(); - for (ContentTag tag : tags) { - Content content = tag.getContent(); + for (Tag tag : pagedTags) { + ContentTag contentTag = (ContentTag) tag; + Content content = contentTag.getContent(); String contentPath = content.getUniquePath(); AbstractFile file = content instanceof AbstractFile ? (AbstractFile) content : null; @@ -220,14 +260,14 @@ public class TagsDAO { content.getName(), null, // GVDTODO translation column contentPath, - tag.getComment(), + contentTag.getComment(), file != null ? TimeZoneUtils.getFormattedTime(file.getMtime()) : "", file != null ? TimeZoneUtils.getFormattedTime(file.getCtime()) : "", file != null ? TimeZoneUtils.getFormattedTime(file.getAtime()) : "", file != null ? TimeZoneUtils.getFormattedTime(file.getCrtime()) : "", content.getSize(), file != null ? StringUtils.defaultString(file.getMd5Hash()) : "", - tag.getUserName()); + contentTag.getUserName()); fileRows.add(new BaseRowDTO( cellValues, @@ -235,6 +275,44 @@ public class TagsDAO { file.getId())); } - return new BaseSearchResultsDTO(FILE_TAG_TYPE_ID, Bundle.FileTag_name_text(), FILE_TAG_COLUMNS, fileRows, 0, fileRows.size()); + return new BaseSearchResultsDTO(FILE_TAG_TYPE_ID, Bundle.FileTag_name_text(), FILE_TAG_COLUMNS, fileRows, 0, allTags.size()); } + + /** + * Handles fetching and paging of data for allTags. + */ + public static class TagFetcher extends DAOFetcher { + + /** + * Main constructor. + * + * @param params Parameters to handle fetching of data. + */ + public TagFetcher(TagsSearchParams params) { + super(params); + } + + @Override + public SearchResultsDTO getSearchResults(int pageSize, int pageIdx, boolean hardRefresh) throws ExecutionException { + return MainDAO.getInstance().getTagsDAO().getTags(this.getParameters(), pageIdx * pageSize, (long) pageSize, hardRefresh); + } + + @Override + public boolean isRefreshRequired(PropertyChangeEvent evt) { + TagsSearchParams params = this.getParameters(); + String eventType = evt.getPropertyName(); + if (eventType.equals(Case.Events.BLACKBOARD_ARTIFACT_TAG_ADDED.toString()) + || eventType.equals(Case.Events.BLACKBOARD_ARTIFACT_TAG_DELETED.toString())) { + + return params.getTagType() == TagsSearchParams.TagType.RESULT; + } + + if (eventType.equals(Case.Events.CONTENT_TAG_ADDED.toString()) + || eventType.equals(Case.Events.CONTENT_TAG_DELETED.toString())) { + + return params.getTagType() == TagsSearchParams.TagType.FILE; + } + return false; + } + } } From d7658d4aff0aa09814f5d61a46f11d504fe6a69a Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 1 Nov 2021 12:56:00 -0400 Subject: [PATCH 11/12] Added cache invalidation logic --- .../autopsy/mainui/datamodel/TagsDAO.java | 62 ++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java index 5dcf62a35a..8050bd670e 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java @@ -36,8 +36,13 @@ import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.casemodule.events.BlackBoardArtifactTagAddedEvent; +import org.sleuthkit.autopsy.casemodule.events.BlackBoardArtifactTagDeletedEvent; +import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent; +import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.TimeZoneUtils; +import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.mainui.nodes.DAOFetcher; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -301,18 +306,69 @@ public class TagsDAO { public boolean isRefreshRequired(PropertyChangeEvent evt) { TagsSearchParams params = this.getParameters(); String eventType = evt.getPropertyName(); + + // handle artifact/result tag changes if (eventType.equals(Case.Events.BLACKBOARD_ARTIFACT_TAG_ADDED.toString()) || eventType.equals(Case.Events.BLACKBOARD_ARTIFACT_TAG_DELETED.toString())) { - return params.getTagType() == TagsSearchParams.TagType.RESULT; + // ignore non-artifact/result tag changes + if (params.getTagType() != TagsSearchParams.TagType.RESULT) { + return false; + } + + if (evt instanceof AutopsyEvent) { + if (evt instanceof BlackBoardArtifactTagAddedEvent) { + // An artifact associated with the current case has been tagged. + BlackBoardArtifactTagAddedEvent event = (BlackBoardArtifactTagAddedEvent) evt; + // ensure tag added event has a valid content id + if (event.getAddedTag() == null || event.getAddedTag().getContent() == null || event.getAddedTag().getArtifact() == null) { + return false; + } + return params.getTagName().getId() == event.getAddedTag().getId(); + } else if (evt instanceof BlackBoardArtifactTagDeletedEvent) { + // A tag has been removed from an artifact associated with the current case. + BlackBoardArtifactTagDeletedEvent event = (BlackBoardArtifactTagDeletedEvent) evt; + // ensure tag deleted event has a valid content id + BlackBoardArtifactTagDeletedEvent.DeletedBlackboardArtifactTagInfo deletedTagInfo = event.getDeletedTagInfo(); + if (deletedTagInfo == null) { + return false; + } + return params.getTagName().getId() == deletedTagInfo.getTagID(); + } + } } + // handle file/content tag changes if (eventType.equals(Case.Events.CONTENT_TAG_ADDED.toString()) || eventType.equals(Case.Events.CONTENT_TAG_DELETED.toString())) { - return params.getTagType() == TagsSearchParams.TagType.FILE; + // ignore non-file/content tag changes + if (params.getTagType() != TagsSearchParams.TagType.FILE) { + return false; + } + + if (evt instanceof AutopsyEvent) { + if (evt instanceof ContentTagAddedEvent) { + // Content associated with the current case has been tagged. + ContentTagAddedEvent event = (ContentTagAddedEvent) evt; + // ensure tag added event has a valid content id + if (event.getAddedTag() == null || event.getAddedTag().getContent() == null) { + return false; + } + return params.getTagName().getId() == event.getAddedTag().getId(); + } else if (evt instanceof ContentTagDeletedEvent) { + // A tag has been removed from content associated with the current case. + ContentTagDeletedEvent event = (ContentTagDeletedEvent) evt; + // ensure tag deleted event has a valid content id + ContentTagDeletedEvent.DeletedContentTagInfo deletedTagInfo = event.getDeletedTagInfo(); + if (deletedTagInfo == null) { + return false; + } + return params.getTagName().getId() == deletedTagInfo.getTagID(); + } + } } return false; } - } + } } From 3b1172cd1c9e94a41a00c5d1a8b9794870ccd006 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 1 Nov 2021 14:35:08 -0400 Subject: [PATCH 12/12] Minor --- .../autopsy/mainui/datamodel/FileTypeSizeSearchParams.java | 4 ++-- Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java index bba60bcb7c..9801467ec6 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/FileTypeSizeSearchParams.java @@ -71,8 +71,8 @@ public class FileTypeSizeSearchParams { @Override public int hashCode() { int hash = 7; - hash = 23 * hash + Objects.hashCode(this.sizeFilter); - hash = 23 * hash + Objects.hashCode(this.dataSourceId); + hash = 53 * hash + Objects.hashCode(this.sizeFilter); + hash = 53 * hash + Objects.hashCode(this.dataSourceId); return hash; } diff --git a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java index 8050bd670e..46a706ebde 100755 --- a/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java +++ b/Core/src/org/sleuthkit/autopsy/mainui/datamodel/TagsDAO.java @@ -24,9 +24,7 @@ import java.beans.PropertyChangeEvent; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; -import java.util.EnumSet; import java.util.List; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -45,7 +43,6 @@ import org.sleuthkit.autopsy.coreutils.TimeZoneUtils; import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.mainui.nodes.DAOFetcher; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifactTag; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag;