From 0ab8f0d76dc0f93ca07ab93039177a6fd5696915 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 12 Sep 2019 17:27:26 -0400 Subject: [PATCH 01/10] 5338 - initial code to optionally limit multi-user access --- .../autopsy/casemodule/AccessLimiter.java | 49 +++++++++++++++++++ .../sleuthkit/autopsy/casemodule/Case.java | 2 +- .../casemodule/NewCaseVisualPanel1.java | 2 +- .../dsp/Bundle.properties-MERGED | 4 +- 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiter.java diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiter.java b/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiter.java new file mode 100644 index 0000000000..9a84ac4657 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiter.java @@ -0,0 +1,49 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule; + +import java.io.File; +import java.nio.file.Paths; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; + +/** + * Class for methods to check if access should be limited to a feature + * + */ +final class AccessLimiter { + + private final static String MULTI_USER_ACCESS_FILE_NAME = "mualimit"; // NON-NLS + private final static String MULTI_USER_ACCESS_FILE_PATH = Paths.get(PlatformUtil.getUserConfigDirectory(), MULTI_USER_ACCESS_FILE_NAME).toString(); + + /** + * Check if privileges regarding multi-user cases should be restricted. + * + * @return True if privileges should be restricted, false otherwise. + */ + static boolean limitMultiUserAccess() { + return new File(MULTI_USER_ACCESS_FILE_PATH).exists(); + } + + /** + * Private constructor for a utility class + */ + private AccessLimiter() { + //private constructer left empty intentionally + } +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index f455a0c9fb..2fb17fdd36 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -1049,7 +1049,7 @@ public class Case { /* * Enable the case-specific actions. */ - CallableSystemAction.get(AddImageAction.class).setEnabled(true); + CallableSystemAction.get(AddImageAction.class).setEnabled(Case.getCurrentCase().getMetadata().getCaseType() == CaseType.SINGLE_USER_CASE || !AccessLimiter.limitMultiUserAccess()); CallableSystemAction.get(CaseCloseAction.class).setEnabled(true); CallableSystemAction.get(CaseDetailsAction.class).setEnabled(true); CallableSystemAction.get(DataSourceSummaryAction.class).setEnabled(true); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java b/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java index aedbb3671f..f49f91a490 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java @@ -61,7 +61,7 @@ final class NewCaseVisualPanel1 extends JPanel implements DocumentListener { */ void readSettings() { caseNameTextField.setText(""); - if (UserPreferences.getIsMultiUserModeEnabled()) { + if (UserPreferences.getIsMultiUserModeEnabled() && !AccessLimiter.limitMultiUserAccess()) { multiUserCaseRadioButton.setEnabled(true); multiUserCaseRadioButton.setSelected(true); } else { diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index 6636cdda9f..21ed945bd2 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -4,11 +4,11 @@ # {0} - file number # {1} - total files -AddLogicalImageTask.addingExtractedFile=Adding extracted file ({0}/{1}) +AddLogicalImageTask.addingExtractedFile=Adding extracted files ({0}/{1}) AddLogicalImageTask.addingExtractedFiles=Adding extracted files # {0} - file number # {1} - total files -AddLogicalImageTask.addingInterestingFile=Adding interesting file ({0}/{1}) +AddLogicalImageTask.addingInterestingFile=Adding interesting files ({0}/{1}) AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files # {0} - file AddLogicalImageTask.addingToReport=Adding {0} to report From 861a47fe6a1acb128421670d6435ff947263eca5 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 12 Sep 2019 17:42:17 -0400 Subject: [PATCH 02/10] 5338 undo accidental commit of merged file unrelated to this PR --- .../autopsy/logicalimager/dsp/Bundle.properties-MERGED | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index 21ed945bd2..6636cdda9f 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -4,11 +4,11 @@ # {0} - file number # {1} - total files -AddLogicalImageTask.addingExtractedFile=Adding extracted files ({0}/{1}) +AddLogicalImageTask.addingExtractedFile=Adding extracted file ({0}/{1}) AddLogicalImageTask.addingExtractedFiles=Adding extracted files # {0} - file number # {1} - total files -AddLogicalImageTask.addingInterestingFile=Adding interesting files ({0}/{1}) +AddLogicalImageTask.addingInterestingFile=Adding interesting file ({0}/{1}) AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files # {0} - file AddLogicalImageTask.addingToReport=Adding {0} to report From e41e53b478c422a8c280fdb93a8c9a672d131d42 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 17 Sep 2019 15:56:32 -0400 Subject: [PATCH 03/10] 5338 rename AccessLimiter class to satisy codacy complaint --- .../{AccessLimiter.java => AccessLimiterUtils.java} | 4 ++-- Core/src/org/sleuthkit/autopsy/casemodule/Case.java | 2 +- .../org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename Core/src/org/sleuthkit/autopsy/casemodule/{AccessLimiter.java => AccessLimiterUtils.java} (95%) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiter.java b/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiterUtils.java similarity index 95% rename from Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiter.java rename to Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiterUtils.java index 9a84ac4657..3b2a201e07 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiter.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AccessLimiterUtils.java @@ -26,7 +26,7 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; * Class for methods to check if access should be limited to a feature * */ -final class AccessLimiter { +final class AccessLimiterUtils { private final static String MULTI_USER_ACCESS_FILE_NAME = "mualimit"; // NON-NLS private final static String MULTI_USER_ACCESS_FILE_PATH = Paths.get(PlatformUtil.getUserConfigDirectory(), MULTI_USER_ACCESS_FILE_NAME).toString(); @@ -43,7 +43,7 @@ final class AccessLimiter { /** * Private constructor for a utility class */ - private AccessLimiter() { + private AccessLimiterUtils() { //private constructer left empty intentionally } } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 2fb17fdd36..0a4586dd42 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -1049,7 +1049,7 @@ public class Case { /* * Enable the case-specific actions. */ - CallableSystemAction.get(AddImageAction.class).setEnabled(Case.getCurrentCase().getMetadata().getCaseType() == CaseType.SINGLE_USER_CASE || !AccessLimiter.limitMultiUserAccess()); + CallableSystemAction.get(AddImageAction.class).setEnabled(Case.getCurrentCase().getMetadata().getCaseType() == CaseType.SINGLE_USER_CASE || !AccessLimiterUtils.limitMultiUserAccess()); CallableSystemAction.get(CaseCloseAction.class).setEnabled(true); CallableSystemAction.get(CaseDetailsAction.class).setEnabled(true); CallableSystemAction.get(DataSourceSummaryAction.class).setEnabled(true); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java b/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java index f49f91a490..789913d78a 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java @@ -61,7 +61,7 @@ final class NewCaseVisualPanel1 extends JPanel implements DocumentListener { */ void readSettings() { caseNameTextField.setText(""); - if (UserPreferences.getIsMultiUserModeEnabled() && !AccessLimiter.limitMultiUserAccess()) { + if (UserPreferences.getIsMultiUserModeEnabled() && !AccessLimiterUtils.limitMultiUserAccess()) { multiUserCaseRadioButton.setEnabled(true); multiUserCaseRadioButton.setSelected(true); } else { From 5050825193cf2d8a35e1cb2a7a4e6a0e3954cd14 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 18 Sep 2019 11:33:47 -0400 Subject: [PATCH 04/10] Make adding interesting files faster. --- .../dsp/AddLogicalImageTask.java | 157 ++++++++++++------ .../dsp/Bundle.properties-MERGED | 3 + 2 files changed, 109 insertions(+), 51 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 6c0bc5a0d6..67dfdccbdf 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -29,6 +29,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.logging.Level; @@ -198,7 +199,8 @@ final class AddLogicalImageTask implements Runnable { } List newDataSources = new ArrayList<>(); - + Map> interestingFileMap = new HashMap<>(); + if (imagePaths.isEmpty()) { createVHD = false; // No VHD in src directory, try ingest the root directory as local files @@ -214,7 +216,7 @@ final class AddLogicalImageTask implements Runnable { try { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingExtractedFiles()); - addExtractedFiles(dest, resultsPath, newDataSources); + interestingFileMap = addExtractedFiles(dest, resultsPath, newDataSources); progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingExtractedFiles()); } catch (IOException | TskCoreException ex) { errorList.add(ex.getMessage()); @@ -241,6 +243,14 @@ final class AddLogicalImageTask implements Runnable { callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, addMultipleImagesTask.getErrorMessages(), emptyDataSources); return; } + try { + interestingFileMap = getInterestingFileMapForVHD(Paths.get(dest.toString(), resultsFilename)); + } catch (TskCoreException | IOException ex) { + errorList.add(Bundle.AddLogicalImageTask_failedToAddInterestingFiles(ex.getMessage())); + LOGGER.log(Level.SEVERE, "Failed to add interesting files", ex); // NON-NLS + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.NONCRITICAL_ERRORS, errorList, emptyDataSources); + } + } catch (NoCurrentCaseException ex) { String msg = Bundle.AddLogicalImageTask_noCurrentCase(); errorList.add(msg); @@ -261,7 +271,7 @@ final class AddLogicalImageTask implements Runnable { try { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFiles()); - addInterestingFiles(Paths.get(dest.toString(), resultsFilename), createVHD); + addInterestingFiles(interestingFileMap); progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingInterestingFiles()); if (createVHD) { callback.done(addMultipleImagesTask.getResult(), addMultipleImagesTask.getErrorMessages(), addMultipleImagesTask.getNewDataSources()); @@ -332,13 +342,67 @@ final class AddLogicalImageTask implements Runnable { "# {0} - target image path", "AddLogicalImageTask.cannotFindDataSourceObjId=Cannot find obj_id in tsk_image_names for {0}", "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingInterestingFile=Adding interesting files ({0}/{1})" }) - private void addInterestingFiles(Path resultsPath, boolean createVHD) throws IOException, TskCoreException { + private void addInterestingFiles(Map> interestingFileMap) throws IOException, TskCoreException { + int lineNumber = 0; + List artifacts = new ArrayList<>(); + + Iterator>> iterator = interestingFileMap.entrySet().iterator(); + while (iterator.hasNext()) { + + if (cancelled) { + // Don't delete destination directory once we started adding interesting files. + // At this point the database and destination directory are complete. + break; + } + + Map.Entry> entry = iterator.next(); + String key = entry.getKey(); + String ruleSetName; + String ruleName; + String[] split = key.split("\t"); + ruleSetName = split[0]; + ruleName = split[1]; + + List fileIds = entry.getValue(); + for (Long fileId: fileIds) { + if (lineNumber % 100 == 0) { + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); + } + addInterestingFileToArtifacts(fileId, ruleSetName, ruleName, artifacts); + lineNumber++; + } + iterator.remove(); + } + + try { + // index the artifact for keyword search + blackboard.postArtifacts(artifacts, MODULE_NAME); + } catch (Blackboard.BlackboardException ex) { + LOGGER.log(Level.SEVERE, "Unable to post artifacts to blackboard", ex); //NON-NLS + } + } + + private void addInterestingFileToArtifacts(long fileId, String ruleSetName, String ruleName, List artifacts) throws TskCoreException { + Collection attributes = new ArrayList<>(); + BlackboardAttribute setNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, ruleSetName); + attributes.add(setNameAttribute); + BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY, MODULE_NAME, ruleName); + attributes.add(ruleNameAttribute); + BlackboardArtifact artifact = this.currentCase.getSleuthkitCase().newBlackboardArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, fileId); + artifact.addAttributes(attributes); + artifacts.add(artifact); + } + + @Messages({ + "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.searchingInterestingFile=Searching for interesting files ({0}/{1})" + }) + private Map> getInterestingFileMapForVHD(Path resultsPath) throws TskCoreException, IOException { Map> objIdToimagePathsMap = currentCase.getSleuthkitCase().getImagePaths(); imagePathToObjIdMap = imagePathsToDataSourceObjId(objIdToimagePathsMap); - + Map> interestingFileMap = new HashMap<>(); + try (BufferedReader br = new BufferedReader(new InputStreamReader( new FileInputStream(resultsPath.toFile()), "UTF8"))) { // NON-NLS - List artifacts = new ArrayList<>(); String line; br.readLine(); // skip the header line int lineNumber = 2; @@ -363,47 +427,36 @@ final class AddLogicalImageTask implements Runnable { String parentPath = fields[8]; if (lineNumber % 100 == 0) { - progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_searchingInterestingFile(lineNumber, totalFiles)); } - String query = makeQuery(createVHD, vhdFilename, fileMetaAddressStr, parentPath, filename); - // TODO - findAllFilesWhere should SQL-escape the query + String query = makeQuery(vhdFilename, fileMetaAddressStr, parentPath, filename); List matchedFiles = Case.getCurrentCase().getSleuthkitCase().findAllFilesWhere(query); + List fileIds = new ArrayList<>(); for (AbstractFile file : matchedFiles) { - addInterestingFileToArtifacts(file, ruleSetName, ruleName, artifacts); + fileIds.add(file.getId()); } + String key = String.format("%s\t%s", ruleSetName, ruleName); + List value = new ArrayList<>(); + if (interestingFileMap.containsKey(key)) { + value = interestingFileMap.get(key); + } + value.addAll(fileIds); + interestingFileMap.put(key, value); lineNumber++; - } // end reading file - - try { - // index the artifact for keyword search - blackboard.postArtifacts(artifacts, MODULE_NAME); - } catch (Blackboard.BlackboardException ex) { - LOGGER.log(Level.SEVERE, "Unable to post artifacts to blackboard", ex); //NON-NLS - } - } - } - - private void addInterestingFileToArtifacts(AbstractFile file, String ruleSetName, String ruleName, List artifacts) throws TskCoreException { - Collection attributes = new ArrayList<>(); - BlackboardAttribute setNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, ruleSetName); - attributes.add(setNameAttribute); - BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY, MODULE_NAME, ruleName); - attributes.add(ruleNameAttribute); - if (!blackboard.artifactExists(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, attributes)) { - BlackboardArtifact artifact = this.currentCase.getSleuthkitCase().newBlackboardArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, file.getId()); - artifact.addAttributes(attributes); - artifacts.add(artifact); + } // end reading file } } + return interestingFileMap; } @Messages({ "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingExtractedFile=Adding extracted files ({0}/{1})" }) - private void addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException, IOException { + private Map> addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException, IOException { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); SleuthkitCase.CaseDbTransaction trans = null; - + Map> interestingFileMap = new HashMap<>(); + try { trans = skCase.beginTransaction(); LocalFilesDataSource localFilesDataSource = skCase.addLocalFilesDataSource(deviceId, this.src.getName(), timeZone, trans); @@ -417,7 +470,7 @@ final class AddLogicalImageTask implements Runnable { while ((line = br.readLine()) != null) { if (cancelled) { rollbackTransaction(trans); - return; + return new HashMap<>(); } String[] fields = line.split("\t", -1); // NON-NLS if (fields.length != 14) { @@ -428,8 +481,8 @@ final class AddLogicalImageTask implements Runnable { // String fileSystemOffsetStr = fields[1]; // String fileMetaAddressStr = fields[2]; // String extractStatusStr = fields[3]; -// String ruleSetName = fields[4]; -// String ruleName = fields[5]; + String ruleSetName = fields[4]; + String ruleName = fields[5]; // String description = fields[6]; String filename = fields[7]; String parentPath = fields[8]; @@ -445,7 +498,7 @@ final class AddLogicalImageTask implements Runnable { } //addLocalFile here - fileImporter.addLocalFile( + AbstractFile fileAdded = fileImporter.addLocalFile( Paths.get(src.toString(), extractedFilePath).toFile(), filename, parentPath, @@ -454,12 +507,19 @@ final class AddLogicalImageTask implements Runnable { Long.parseLong(atime), Long.parseLong(mtime), localFilesDataSource); - + String key = String.format("%s\t%s", ruleSetName, ruleName); + List value = new ArrayList<>(); + if (interestingFileMap.containsKey(key)) { + value = interestingFileMap.get(key); + } + value.add(fileAdded.getId()); + interestingFileMap.put(key, value); lineNumber++; } // end reading file } trans.commit(); newDataSources.add(localFilesDataSource); + return interestingFileMap; } catch (NumberFormatException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error adding extracted files", ex); // NON-NLS @@ -489,21 +549,16 @@ final class AddLogicalImageTask implements Runnable { } } - String makeQuery(boolean createVHD, String vhdFilename, String fileMetaAddressStr, String parentPath, String filename) throws TskCoreException { + String makeQuery(String vhdFilename, String fileMetaAddressStr, String parentPath, String filename) throws TskCoreException { String query; - if (createVHD) { - String targetImagePath = Paths.get(dest.toString(), vhdFilename).toString(); - Long dataSourceObjId = imagePathToObjIdMap.get(targetImagePath); - if (dataSourceObjId == null) { - throw new TskCoreException(Bundle.AddLogicalImageTask_cannotFindDataSourceObjId(targetImagePath)); - } - query = String.format("data_source_obj_id = '%s' AND meta_addr = '%s' AND name = '%s'", // NON-NLS - dataSourceObjId.toString(), fileMetaAddressStr, filename.replace("'", "''")); - } else { - String newParentPath = "/" + ROOT_STR + "/" + vhdFilename + "/" + parentPath; - query = String.format("name = '%s' AND parent_path = '%s'", // NON-NLS - filename.replace("'", "''"), newParentPath.replace("'", "''")); + String targetImagePath = Paths.get(dest.toString(), vhdFilename).toString(); + Long dataSourceObjId = imagePathToObjIdMap.get(targetImagePath); + if (dataSourceObjId == null) { + throw new TskCoreException(Bundle.AddLogicalImageTask_cannotFindDataSourceObjId(targetImagePath)); } + query = String.format("data_source_obj_id = '%s' AND meta_addr = '%s' AND name = '%s'", // NON-NLS + dataSourceObjId.toString(), fileMetaAddressStr, filename.replace("'", "''")); + // TODO - findAllFilesWhere should SQL-escape the query return query; } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index 3e44d06104..b2e284c666 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -46,6 +46,9 @@ AddLogicalImageTask.noCurrentCase=No current case # {1} - fields length # {2} - expected length AddLogicalImageTask.notEnoughFields=File does not contain enough fields at line {0}, got {1}, expecting {2} +# {0} - file number +# {1} - total files +AddLogicalImageTask.searchingInterestingFile=Searching for interesting files ({0}/{1}) # {0} - imageFilePath AddMultipleImagesTask.adding=Adding: {0} # {0} - file From cc1c5b283ad53b9691727a3beb2497cab483ecbe Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 18 Sep 2019 15:03:14 -0400 Subject: [PATCH 05/10] Fix PR comment --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 67dfdccbdf..be85936754 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -437,12 +437,11 @@ final class AddLogicalImageTask implements Runnable { fileIds.add(file.getId()); } String key = String.format("%s\t%s", ruleSetName, ruleName); - List value = new ArrayList<>(); if (interestingFileMap.containsKey(key)) { - value = interestingFileMap.get(key); + interestingFileMap.get(key).addAll(fileIds); + } else { + interestingFileMap.put(key, fileIds); } - value.addAll(fileIds); - interestingFileMap.put(key, value); lineNumber++; } // end reading file } } From 11dbce81f29de9b5ad74a2e8df48a643fc072f5e Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 18 Sep 2019 16:42:24 -0400 Subject: [PATCH 06/10] modified vcard code --- .../thunderbirdparser/VcardParser.java | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java index 84f4cd92c8..3cc4f64e64 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java @@ -224,8 +224,6 @@ final class VcardParser { if (!tskBlackboard.artifactExists(abstractFile, BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT, attributes)) { artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); artifact.addAttributes(attributes); - List blackboardArtifacts = new ArrayList<>(); - blackboardArtifacts.add(artifact); extractPhotos(vcard, abstractFile, artifact); @@ -388,8 +386,12 @@ final class VcardParser { */ private void addPhoneAttributes(Telephone telephone, AbstractFile abstractFile, Collection attributes) { String telephoneText = telephone.getText(); + if (telephoneText == null || telephoneText.isEmpty()) { - return; + telephoneText = telephone.getUri().getNumber(); + if (telephoneText == null || telephoneText.isEmpty()) { + return; + } } // Add phone number to collection for later creation of TSK_CONTACT. @@ -408,20 +410,25 @@ final class VcardParser { type.getValue().toUpperCase().replaceAll("\\s+","").split(",")); for (String splitType : splitTelephoneTypes) { - String attributeTypeName = "TSK_PHONE_NUMBER_" + splitType; + String attributeTypeName = "TSK_PHONE_NUMBER"; + if(splitType != null && !splitType.isEmpty()) { + attributeTypeName = "TSK_PHONE_NUMBER_" + splitType; + } + try { BlackboardAttribute.Type attributeType = tskCase.getAttributeType(attributeTypeName); if (attributeType == null) { // Add this attribute type to the case database. - attributeType = tskCase.addArtifactAttributeType(attributeTypeName, - BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, - String.format("Phone (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + attributeType = tskCase.addArtifactAttributeType(attributeTypeName, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, + String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + } - ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephone.getText(), attributeType, attributes); + ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); } catch (TskDataException ex) { - logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + logger.log(Level.WARNING, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); } } } @@ -490,7 +497,11 @@ final class VcardParser { private void addPhoneAccountInstances(Telephone telephone, AbstractFile abstractFile, Collection accountInstances) { String telephoneText = telephone.getText(); if (telephoneText == null || telephoneText.isEmpty()) { - return; + telephoneText = telephone.getUri().getNumber(); + if (telephoneText == null || telephoneText.isEmpty()) { + return; + } + } // Add phone number as a TSK_ACCOUNT. From 13c379c9a21f2020be280f478ff6468e250b013d Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 19 Sep 2019 09:12:18 -0400 Subject: [PATCH 07/10] Changed ingest module docs to use postArtifact(). Updated link to blackboard. --- docs/doxygen/modIngest.dox | 36 ++++++++++++++----------------- docs/doxygen/platformConcepts.dox | 2 +- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/docs/doxygen/modIngest.dox b/docs/doxygen/modIngest.dox index ff4f051479..29af4ee263 100644 --- a/docs/doxygen/modIngest.dox +++ b/docs/doxygen/modIngest.dox @@ -198,27 +198,23 @@ See \ref platform_blackboard for details on posting results to it. You use the The blackboard defines artifacts for specific data types (such as web bookmarks). You can use one of the standard artifact types or create your own. -When modules add data to the blackboard, they should notify listeners of the new -data by invoking the org.sleuthkit.autopsy.ingest.IngestServices.fireModuleDataEvent() method. -Do so as soon as you have added an artifact to the blackboard. -This allows other modules (and the main UI) to know when to query the blackboard -for the latest data. However, if you are writing a large number of blackboard -artifacts in a loop, it is better to invoke org.sleuthkit.autopsy.ingest.IngestServices.fireModuleDataEvent() -only once after the bulk write, so as not to flood the system with events. +After you make an artifact, you should call sleuthkit.Blackboard.postArtifact(), which will: +
    +
  • Analyze the artifact and add any timestamps to the new timeline tables +
  • Send an event over its new EventBus that the artifact(s) was added. +
      +
    • Autopsy is a listener of this new eventbus and will rebroadcast it to other Autopsy modules +
    • Keyword search will also listen for this and will index the artifact +
    +
-Further, when modules create artifacts, they should be indexed for keyword search, -using the method org.sleuthkit.autopsy.casemodule.services.Blackboard.indexArtifact(BlackboardArtifact artifact). This can be done -in the following way: - -\code -Blackboard blackboard = Case.getCurrentCase().getServices().getBlackboard(); -try { - blackboard.indexArtifact(artifact); //Your artifact as the argument. -} -catch (BlackboardException ex) { - //YOUR EXCEPTION BEHAVIOR HERE. -} -\endcode +This means you no longer have to make calls to: + - Index the artifact + - Fire the event to refresh the UI. + +If you are creating a large number of artifacts, you may see better performance if you save all the artifacts and do one bulk write at the end using sleuthkit.Blackboard.postArtifacts() + +You should not be using the Autopsy version of Blackboard. Those methods have all been deprecated and is another example of us moving "services" into the TSK data model. \subsection ingest_modules_making_results_report Making a Report diff --git a/docs/doxygen/platformConcepts.dox b/docs/doxygen/platformConcepts.dox index af202a1cba..23389d935b 100644 --- a/docs/doxygen/platformConcepts.dox +++ b/docs/doxygen/platformConcepts.dox @@ -52,7 +52,7 @@ The blackboard allows modules to communicate with each other and the UI. It has The blackboard is not unique to Autopsy. It is part of The Sleuth Kit datamodel and The Sleuth Kit Framework. In the name of reducing the amount of documentation that we need to maintain, we provide links here to those documentation sources. -- The Blackboard +- The Blackboard \subsection mod_dev_other_services Framework Services and Utilities From 1c2ace3537a6dd51f24cef689dc8cd3f9ba88b6d Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 19 Sep 2019 09:15:31 -0400 Subject: [PATCH 08/10] Wording changes --- docs/doxygen/modIngest.dox | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/doxygen/modIngest.dox b/docs/doxygen/modIngest.dox index 29af4ee263..3c6ea81ebb 100644 --- a/docs/doxygen/modIngest.dox +++ b/docs/doxygen/modIngest.dox @@ -200,10 +200,10 @@ You can use one of the standard artifact types or create your own. After you make an artifact, you should call sleuthkit.Blackboard.postArtifact(), which will:
    -
  • Analyze the artifact and add any timestamps to the new timeline tables -
  • Send an event over its new EventBus that the artifact(s) was added. +
  • Analyze the artifact and add any timestamps to the Timeline tables +
  • Send an event over the EventBus that the artifact(s) was added
      -
    • Autopsy is a listener of this new eventbus and will rebroadcast it to other Autopsy modules +
    • Autopsy is a listener of this EventBus and will rebroadcast it to other Autopsy modules
    • Keyword search will also listen for this and will index the artifact
From d14dae01f9a3fb5adc92ad579d92d67dfc0d66a6 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 19 Sep 2019 09:44:37 -0400 Subject: [PATCH 09/10] Rewording --- docs/doxygen/modIngest.dox | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/doxygen/modIngest.dox b/docs/doxygen/modIngest.dox index 3c6ea81ebb..a2d81627b9 100644 --- a/docs/doxygen/modIngest.dox +++ b/docs/doxygen/modIngest.dox @@ -192,27 +192,27 @@ The first question that you must answer is what type of data do you want the use -# Data that is in a big text file or some other report that the user can review. To do this, you will use the Case.addReport() method to make the output available in the directory tree. -\subsection ingest_modules_making_results_bb Posting Results to the Blackboard +\subsection ingest_modules_making_results_bb Saving Results to the Blackboard The blackboard is used to store results so that they are displayed in the results tree. -See \ref platform_blackboard for details on posting results to it. You use the blackboard when you have specific items to show the user. if you want to just shown them a big report from another library or tool, see \ref mod_report_page. +See \ref platform_blackboard for details on saving results to it. You use the blackboard when you have specific items to show the user. If you want to just shown them a big report from another library or tool, see \ref mod_report_page. The blackboard defines artifacts for specific data types (such as web bookmarks). You can use one of the standard artifact types or create your own. -After you make an artifact, you should call sleuthkit.Blackboard.postArtifact(), which will: +After you've added an artifact and all of its attributes to the blackboard, you should call sleuthkit.Blackboard.postArtifact(), which will:
  • Analyze the artifact and add any timestamps to the Timeline tables -
  • Send an event over the EventBus that the artifact(s) was added +
  • Send an event over the Sleuth Kit event bus that the artifact(s) was added
      -
    • Autopsy is a listener of this EventBus and will rebroadcast it to other Autopsy modules -
    • Keyword search will also listen for this and will index the artifact +
    • Autopsy is a listener of this event bus and will rebroadcast the event to other Autopsy modules +
    • Keyword search also listens for this event and will index the artifact
-This means you no longer have to make calls to: +This means you no longer have to make separate calls to: - Index the artifact - Fire the event to refresh the UI. -If you are creating a large number of artifacts, you may see better performance if you save all the artifacts and do one bulk write at the end using sleuthkit.Blackboard.postArtifacts() +If you are creating a large number of artifacts, you may see better performance if you save all the artifacts you create and do one bulk post at the end using sleuthkit.Blackboard.postArtifacts(). You can also post batches of artifacts instead of saving all of them until the end. You should not be using the Autopsy version of Blackboard. Those methods have all been deprecated and is another example of us moving "services" into the TSK data model. From 9c0308cbc1636fc2ac94f6b6ca0aa39e4c95450f Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 19 Sep 2019 12:05:01 -0400 Subject: [PATCH 10/10] Fixed timeline bug --- Core/src/org/sleuthkit/autopsy/timeline/ui/ViewFrame.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/ViewFrame.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/ViewFrame.java index ea0da29822..9e780277f3 100755 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/ViewFrame.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/ViewFrame.java @@ -770,8 +770,11 @@ final public class ViewFrame extends BorderPane { Notifications.create().owner(getScene().getWindow()) .text(Bundle.ViewFrame_pickerListener_errorMessage()) .showError(); - logger.log(Level.SEVERE, "Error responding to date/time picker change.", ex); + logger.log(Level.WARNING, "Error responding to date/time picker change.", ex); //NON-NLS + } catch (IllegalArgumentException ex ) { + logger.log(Level.INFO, "Timeline: User supplied invalid time range."); //NON-NLS } + Platform.runLater(ViewFrame.this::refreshTimeUI); } }