From 69b6eb14faa98ffce70aefb243b0b65ed3235e0c Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Wed, 6 Jun 2018 18:12:03 -0600 Subject: [PATCH] more progress --- ...ralRepositoryCaseFileInstanceMetadata.java | 46 ++++++ .../CommonFilesMetadataBuilder.java | 10 +- .../EamDbCommonFilesAlgorithm.java | 8 +- .../FileInstanceNodeGenerator.java | 132 ++++++++++++++++++ .../autopsy/commonfilesearch/Md5Metadata.java | 14 +- ...=> SleuthkitCaseFileInstanceMetadata.java} | 49 ++----- .../sleuthkit/autopsy/datamodel/Md5Node.java | 35 ++--- .../commonfilessearch/InterCaseUtils.java | 6 +- .../commonfilessearch/IntraCaseUtils.java | 4 +- 9 files changed, 221 insertions(+), 83 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepositoryCaseFileInstanceMetadata.java create mode 100644 Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceNodeGenerator.java rename Core/src/org/sleuthkit/autopsy/commonfilesearch/{FileInstanceMetadata.java => SleuthkitCaseFileInstanceMetadata.java} (50%) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepositoryCaseFileInstanceMetadata.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepositoryCaseFileInstanceMetadata.java new file mode 100644 index 0000000000..2f3a1158be --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepositoryCaseFileInstanceMetadata.java @@ -0,0 +1,46 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.commonfilesearch; + +import java.util.Map; +import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepositoryFile; +import org.sleuthkit.autopsy.datamodel.CentralRepositoryFileInstanceNode; +import org.sleuthkit.autopsy.datamodel.DisplayableItemNode; +import org.sleuthkit.datamodel.AbstractFile; + +/** + * Generates a DisplayableItmeNode using a CentralRepositoryFile. + */ +public class CentralRepositoryCaseFileInstanceMetadata extends FileInstanceNodeGenerator { + + private CentralRepositoryFile crFile; + + CentralRepositoryCaseFileInstanceMetadata(CentralRepositoryFile crFile, Long abstractFileReference, Map cachedFiles, String dataSource){ + super(abstractFileReference, cachedFiles, dataSource); + //TODO should we actually just take an ID instead of the whole object + // like we've done previously, or is this ok? + this.crFile = crFile; + } + + @Override + public DisplayableItemNode generateNode() { + return new CentralRepositoryFileInstanceNode(this.crFile, this.lookupOrCreateAbstractFile()); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadataBuilder.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadataBuilder.java index 41c21f1dff..84333ba521 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadataBuilder.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetadataBuilder.java @@ -32,6 +32,7 @@ import java.util.stream.Stream; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.HashUtility; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase.CaseDbQuery; @@ -172,6 +173,8 @@ public abstract class CommonFilesMetadataBuilder { SleuthkitCase sleuthkitCase = Case.getCurrentCaseThrows().getSleuthkitCase(); String selectStatement = this.buildSqlSelectStatement(); + + Map fileCache = new HashMap<>(); try ( CaseDbQuery query = sleuthkitCase.executeQuery(selectStatement); @@ -189,11 +192,10 @@ public abstract class CommonFilesMetadataBuilder { if (commonFiles.containsKey(md5)) { final Md5Metadata md5Metadata = commonFiles.get(md5); - md5Metadata.addFileInstanceMetadata(new FileInstanceMetadata(objectId, dataSource)); + md5Metadata.addFileInstanceMetadata(new SleuthkitCaseFileInstanceMetadata(objectId, fileCache, dataSource)); } else { - final List fileInstances = new ArrayList<>(); - fileInstances.add(new FileInstanceMetadata(objectId, dataSource)); - Md5Metadata md5Metadata = new Md5Metadata(md5, fileInstances); + final Md5Metadata md5Metadata = new Md5Metadata(md5); + md5Metadata.addFileInstanceMetadata(new SleuthkitCaseFileInstanceMetadata(objectId, fileCache, dataSource)); commonFiles.put(md5, md5Metadata); } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/EamDbCommonFilesAlgorithm.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/EamDbCommonFilesAlgorithm.java index 443f7d98b4..4fc7632919 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/EamDbCommonFilesAlgorithm.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/EamDbCommonFilesAlgorithm.java @@ -111,6 +111,7 @@ public abstract class EamDbCommonFilesAlgorithm extends CommonFilesMetadataBuild private Map gatherIntercaseResults(Collection artifactInstances, Map commonFiles) { Map interCaseCommonFiles = new HashMap<>(); + Map cachedFiles = new HashMap<>(); for (CentralRepositoryFile instance : artifactInstances) { @@ -130,15 +131,16 @@ public abstract class EamDbCommonFilesAlgorithm extends CommonFilesMetadataBuild //TODO resume here!!! //TODO need to figure out if we have a CR instance or a SK resource and create as appropriate.... - Long objectId = commonFiles.get(md5).getMetadata().iterator().next().getObjectId(); + Long objectId = commonFiles.get(md5).getMetadata().iterator().next().getIdenticalFileSleuthkitCaseObjectID(); + if(interCaseCommonFiles.containsKey(md5)) { //Add to intercase metaData final Md5Metadata md5Metadata = interCaseCommonFiles.get(md5); - md5Metadata.addFileInstanceMetadata(new FileInstanceMetadata(objectId, dataSource), correlationCaseDisplayName); + md5Metadata.addFileInstanceMetadata(new SleuthkitCaseFileInstanceMetadata(objectId, dataSource), correlationCaseDisplayName); } else { Md5Metadata md5Metadata = new Md5Metadata(md5); - md5Metadata.addFileInstanceMetadata(new FileInstanceMetadata(objectId, dataSource), correlationCaseDisplayName); + md5Metadata.addFileInstanceMetadata(new SleuthkitCaseFileInstanceMetadata(objectId, dataSource), correlationCaseDisplayName); interCaseCommonFiles.put(md5, md5Metadata); } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceNodeGenerator.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceNodeGenerator.java new file mode 100644 index 0000000000..f4706332b7 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceNodeGenerator.java @@ -0,0 +1,132 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.commonfilesearch; + +import java.util.Map; +import java.util.logging.Level; +import org.openide.util.Exceptions; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepositoryFile; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.datamodel.CentralRepositoryFileInstanceNode; +import org.sleuthkit.autopsy.datamodel.DisplayableItemNode; +import org.sleuthkit.autopsy.datamodel.SleuthkitCaseFileInstanceNode; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Defines types which can be used to get some sort of File Instance node (a + * child of the MD5Node) for use in the common files tree table. + */ +public abstract class FileInstanceNodeGenerator { + + private static final Logger LOGGER = Logger.getLogger(FileInstanceNodeGenerator.class.getName()); + protected Long abstractFileReference; + protected Map cachedFiles; + private String dataSource; + + public FileInstanceNodeGenerator(Long abstractFileReference, Map cachedFiles, String dataSource){ + this.abstractFileReference = abstractFileReference; + this.cachedFiles = cachedFiles; + this.dataSource = dataSource; + } + + /** + * Grab a cached instance of the AbstractFile or grab one from the + * SleuthkitCase. Use this in implementations of generateNode. + * + * @param objectId + * @param cachedFiles + * @return AbstractFile which is identical to the file instance generated by + * implementations of this object + */ + protected AbstractFile lookupOrCreateAbstractFile() { + if (this.cachedFiles.containsKey(this.abstractFileReference)) { + return this.cachedFiles.get(this.abstractFileReference); + } else { + AbstractFile file = FileInstanceNodeGenerator.loadFileFromSleuthkitCase(this.abstractFileReference); + this.cachedFiles.put(this.abstractFileReference, file); + return file; + } + } + + private static AbstractFile loadFileFromSleuthkitCase(Long objectId) { + + Case currentCase; + try { + currentCase = Case.getCurrentCaseThrows(); + + SleuthkitCase tskDb = currentCase.getSleuthkitCase(); + + AbstractFile abstractFile = tskDb.findAllFilesWhere(String.format("obj_id in (%s)", objectId)).get(0); + + return abstractFile; + + } catch (TskCoreException | NoCurrentCaseException ex) { + LOGGER.log(Level.SEVERE, String.format("Unable to find AbstractFile for record with obj_id: %s. Node not created.", new Object[]{objectId}), ex); + return null; + } + } + + private static AbstractFile lookupOrCreateAbstractFile(Long objectId, Map cachedFiles){ + if (cachedFiles.containsKey(objectId)) { + return cachedFiles.get(objectId); + } else { + AbstractFile file = FileInstanceNodeGenerator.loadFileFromSleuthkitCase(objectId); + cachedFiles.put(objectId, file); + return file; + } + } + + /** + * Create a node which is a child of the MD5Node, to be used to display a + * row in the tree table + * + * @return child row node + */ + public abstract DisplayableItemNode generateNode(); + + /** + * Get string name of the data source where this instance appears. + * + * @return + */ + public String getDataSource(){ + + /** + * Even though we could get this from the CR record or the AbstractFile, + * we want to avoid getting it from the AbstractFile because it would be an + * extra database roundtrip. + */ + return this.dataSource; + } + + public Long getIdenticalFileSleuthkitCaseObjectID(){ + return this.abstractFileReference; + } + + public static FileInstanceNodeGenerator createInstance(Long objectId, CentralRepositoryFile instance, Map cachedFiles){ + AbstractFile referenceFile = FileInstanceNodeGenerator.lookupOrCreateAbstractFile(objectId, cachedFiles); + + + } +} diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/Md5Metadata.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/Md5Metadata.java index 6f9e2472e7..2ab8db4ee3 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/Md5Metadata.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/Md5Metadata.java @@ -32,9 +32,9 @@ import java.util.Set; final public class Md5Metadata { private final String md5; - private final List fileInstances; + private final List fileInstances; - Md5Metadata(String md5, List fileInstances){ + Md5Metadata(String md5, List fileInstances){ this.md5 = md5; this.fileInstances = fileInstances; } @@ -48,15 +48,15 @@ final public class Md5Metadata { return this.md5; } - void addFileInstanceMetadata(FileInstanceMetadata metadata){ + void addFileInstanceMetadata(FileInstanceNodeGenerator metadata){ this.fileInstances.add(metadata); } - void addFileInstanceMetadata(FileInstanceMetadata metadata, String caseName){ + void addFileInstanceMetadata(SleuthkitCaseFileInstanceMetadata metadata, String caseName){ this.fileInstances.add(metadata); } - public Collection getMetadata(){ + public Collection getMetadata(){ return Collections.unmodifiableCollection(this.fileInstances); } @@ -70,8 +70,8 @@ final public class Md5Metadata { public String getDataSources() { Set sources = new HashSet<> (); - for(FileInstanceMetadata data : this.fileInstances){ - sources.add(data.getDataSourceName()); + for(FileInstanceNodeGenerator data : this.fileInstances){ + sources.add(data.getDataSource()); } return String.join(", ", sources); } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceMetadata.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SleuthkitCaseFileInstanceMetadata.java similarity index 50% rename from Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceMetadata.java rename to Core/src/org/sleuthkit/autopsy/commonfilesearch/SleuthkitCaseFileInstanceMetadata.java index f27dcafe48..a1139aeb44 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceMetadata.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SleuthkitCaseFileInstanceMetadata.java @@ -20,58 +20,29 @@ package org.sleuthkit.autopsy.commonfilesearch; import java.io.File; +import java.util.Map; import org.openide.nodes.Node; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepositoryFile; +import org.sleuthkit.autopsy.datamodel.DisplayableItemNode; +import org.sleuthkit.autopsy.datamodel.SleuthkitCaseFileInstanceNode; +import org.sleuthkit.datamodel.AbstractFile; /** * Encapsulates data required to instantiate a FileInstanceNode. */ -final public class FileInstanceMetadata { //TODO become abstract or interface - - private final Long objectId; - private final String dataSourceName; - private final CentralRepositoryFile crFile; +final public class SleuthkitCaseFileInstanceMetadata extends FileInstanceNodeGenerator { //TODO become abstract or interface /** * Create meta data required to find an abstract file and build a FileInstanceNode. * @param objectId id of abstract file to find * @param dataSourceName name of datasource where the object is found */ - FileInstanceMetadata (Long objectId, String dataSourceName) { - this.objectId = objectId; - this.dataSourceName = dataSourceName; - this.crFile = null; - } - - FileInstanceMetadata (CentralRepositoryFile crFile){ - //TODO should we actually just take an ID instead of the whole object - // like we've done previously, or is this ok? - this.objectId = null; - this.dataSourceName = null; - this.crFile = crFile; - } - - /** - * obj_id for the file represented by this object - * @return - */ - public Long getObjectId(){ - return this.objectId; - } - - /** - * Name of datasource where this instance was found. - * @return - */ - public String getDataSourceName(){ - return this.dataSourceName; + SleuthkitCaseFileInstanceMetadata (Long abstractFileReference, Map cachedFiles, String dataSource) { + super(abstractFileReference, cachedFiles, dataSource); } - public boolean isPresentInCurrentCase() { - return this.crFile != null; - } - - public CentralRepositoryFile getCentralRepoFileInstance() { - return this.crFile; + @Override + public DisplayableItemNode generateNode() { + return new SleuthkitCaseFileInstanceNode(this.lookupOrCreateAbstractFile(), this.getDataSource()); } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Md5Node.java b/Core/src/org/sleuthkit/autopsy/datamodel/Md5Node.java index 4c7f5a497d..1287fd061f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Md5Node.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Md5Node.java @@ -19,6 +19,7 @@ */ package org.sleuthkit.autopsy.datamodel; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -32,7 +33,8 @@ import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepositoryFile; -import org.sleuthkit.autopsy.commonfilesearch.FileInstanceMetadata; +import org.sleuthkit.autopsy.commonfilesearch.FileInstanceNodeGenerator; +import org.sleuthkit.autopsy.commonfilesearch.SleuthkitCaseFileInstanceMetadata; import org.sleuthkit.autopsy.commonfilesearch.Md5Metadata; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; @@ -130,41 +132,24 @@ public class Md5Node extends DisplayableItemNode { * Child generator for SleuthkitCaseFileInstanceNode of * Md5Node. */ - static class FileInstanceNodeFactory extends ChildFactory { + static class FileInstanceNodeFactory extends ChildFactory { + private Map cachedFiles; + private final Md5Metadata descendants; FileInstanceNodeFactory(Md5Metadata descendants) { this.descendants = descendants; + this.cachedFiles = new HashMap<>(); } @Override - protected Node createNodeForKey(FileInstanceMetadata file) { - - try { - Case currentCase = Case.getCurrentCaseThrows(); - SleuthkitCase tskDb = currentCase.getSleuthkitCase(); - - AbstractFile abstractFile = tskDb.findAllFilesWhere(String.format("obj_id in (%s)", file.getObjectId())).get(0); - - //TODO it would be an improvement to utilize some sort of polymorphism - // (interface with getNode is probably plenty) rather than this conditional - if (file.isPresentInCurrentCase()) { - return new SleuthkitCaseFileInstanceNode(abstractFile, file.getDataSourceName()); - } else { - CentralRepositoryFile crFile = file.getCentralRepoFileInstance(); - return new CentralRepositoryFileInstanceNode(crFile, abstractFile); - } - - } catch (NoCurrentCaseException | TskCoreException ex) { - LOGGER.log(Level.SEVERE, String.format("Unable to find AbstractFile for record with obj_id: %s. Node not created.", new Object[]{file.getObjectId()}), ex); - } - - return null; + protected Node createNodeForKey(FileInstanceNodeGenerator file) { + return file.generateNode(); } @Override - protected boolean createKeys(List list) { + protected boolean createKeys(List list) { list.addAll(this.descendants.getMetadata()); return true; } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/InterCaseUtils.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/InterCaseUtils.java index 4025563d2c..ba8e9dccce 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/InterCaseUtils.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/InterCaseUtils.java @@ -53,7 +53,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata; import org.sleuthkit.autopsy.commonfilesearch.DataSourceLoader; -import org.sleuthkit.autopsy.commonfilesearch.FileInstanceMetadata; +import org.sleuthkit.autopsy.commonfilesearch.SleuthkitCaseFileInstanceMetadata; import org.sleuthkit.autopsy.commonfilesearch.Md5Metadata; import org.sleuthkit.datamodel.AbstractFile; @@ -297,9 +297,9 @@ class InterCaseUtils { for(Map.Entry file : searchDomain.getMetadata().entrySet()){ - Collection fileInstances = file.getValue().getMetadata(); + Collection fileInstances = file.getValue().getMetadata(); - for(FileInstanceMetadata fileInstance : fileInstances){ + for(SleuthkitCaseFileInstanceMetadata fileInstance : fileInstances){ } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java index e372cf7af3..10bd2e2fc4 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java @@ -36,7 +36,7 @@ import org.sleuthkit.autopsy.casemodule.ImageDSProcessor; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata; import org.sleuthkit.autopsy.commonfilesearch.DataSourceLoader; -import org.sleuthkit.autopsy.commonfilesearch.FileInstanceMetadata; +import org.sleuthkit.autopsy.commonfilesearch.SleuthkitCaseFileInstanceMetadata; import org.sleuthkit.autopsy.commonfilesearch.Md5Metadata; import org.sleuthkit.autopsy.testutils.CaseUtils; import org.sleuthkit.autopsy.testutils.IngestUtils; @@ -187,7 +187,7 @@ class IntraCaseUtils { Map instanceIdToDataSource = new HashMap<>(); for (Map.Entry entry : metadata.getMetadata().entrySet()) { - for (FileInstanceMetadata md : entry.getValue().getMetadata()) { + for (SleuthkitCaseFileInstanceMetadata md : entry.getValue().getMetadata()) { instanceIdToDataSource.put(md.getObjectId(), md.getDataSourceName()); } }