From f30ce57af85c575334199f75dabca439c10657b0 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Fri, 26 Mar 2021 10:12:27 -0400 Subject: [PATCH 01/15] Add exception checking for file coming back from java-libpst Checking for parsing errors on file when it comes back from java-libpst --- .../org/sleuthkit/autopsy/thunderbirdparser/PstParser.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java index 9cbc8c6a6f..358c67a8a5 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java @@ -107,6 +107,11 @@ class PstParser implements AutoCloseable{ logger.log(Level.INFO, "Found encrypted PST file."); //NON-NLS return ParseResult.ENCRYPT; } + if (ex.getMessage().toLowerCase().startsWith("unable to")) { + logger.log(Level.WARNING, ex.getMessage()); + logger.log(Level.WARNING, String.format("Error in parsing PST file %s, file may be empty or corrupt", file.getName())); + return ParseResult.ERROR; + } String msg = file.getName() + ": Failed to create internal java-libpst PST file to parse:\n" + ex.getMessage(); //NON-NLS logger.log(Level.WARNING, msg, ex); return ParseResult.ERROR; From d8615de1c03a88ff46df39faae9191b054d042b6 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 29 Mar 2021 16:03:35 -0400 Subject: [PATCH 02/15] domain categorization polish --- .../url/analytics/domaincategorization/Bundle.properties | 2 +- .../domaincategorization/WebCategoriesOptionsPanel.form | 4 ++-- .../domaincategorization/WebCategoriesOptionsPanel.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties index fae4fa97ec..f4d258315a 100644 --- a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties @@ -4,7 +4,7 @@ AddEditCategoryDialog.categoryLabel.text=Category: AddEditCategoryDialog.domainSuffixLabel.text=Domain Suffix: AddEditCategoryDialog.saveButton.text=Save AddEditCategoryDialog.cancelButton.text=Cancel -WebCategoriesOptionsPanel.panelDescription.text=This module allows you to classify web sites based on domain names. +WebCategoriesOptionsPanel.panelDescription.text=This feature allows the Recent Activity ingest module to perform custom categorization of web sites based on domain suffixes. WebCategoriesOptionsPanel.categoriesTitle.text=Categories: WebCategoriesOptionsPanel.newEntryButton.text=New Entry WebCategoriesOptionsPanel.editEntryButton.text=Edit Entry diff --git a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.form b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.form index 177919921c..00dc510d0d 100644 --- a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.form @@ -197,8 +197,8 @@ - - + + diff --git a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java index 3f7ca1297d..7db16e9d19 100644 --- a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java @@ -373,7 +373,7 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i gridBagConstraints.weightx = 1.0; add(bottomStrut, gridBagConstraints); - ingestRunningWarning.setForeground(java.awt.Color.RED); + ingestRunningWarning.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/warning16.png"))); // NOI18N ingestRunningWarning.setText(org.openide.util.NbBundle.getMessage(WebCategoriesOptionsPanel.class, "WebCategoriesOptionsPanel.ingestRunningWarning.text")); // NOI18N gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.gridx = 0; From d3a12cba71c4fc5ae1b8b7fd5d73232879e0b351 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 29 Mar 2021 16:05:44 -0400 Subject: [PATCH 03/15] reset runner --- .../recentactivity/DomainCategoryRunner.java | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java index 0102f6e868..f65673b6d0 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java @@ -32,7 +32,6 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.lang.StringUtils; import org.openide.util.Lookup; import org.openide.util.NbBundle.Messages; @@ -87,17 +86,6 @@ class DomainCategoryRunner extends Extract { // NOTE: if CustomWebCategorizer ever changes name, this will need to be changed as well. private static final String CUSTOM_CATEGORIZER_PATH = "org.sleuthkit.autopsy.url.analytics.domaincategorization.CustomWebCategorizer"; - // the artifact types to be searched for domain categories - private static final List DOMAIN_CATEGORIZATION_TYPES = Stream.of( - BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK, - BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_CACHE, - BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE, - BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD, - BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, - BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY) - .map(BlackboardArtifact.Type::new) - .collect(Collectors.toList()); - /** * Get seconds from epoch from the mapping for the attribute type id. * @@ -180,7 +168,7 @@ class DomainCategoryRunner extends Extract { * Main constructor. */ DomainCategoryRunner() { - + } /** @@ -367,7 +355,7 @@ class DomainCategoryRunner extends Extract { Set hostSuffixesSeen = new HashSet<>(); try { List listArtifacts = currentCase.getSleuthkitCase().getBlackboard().getArtifacts( - DOMAIN_CATEGORIZATION_TYPES, + Arrays.asList(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_HISTORY)), Arrays.asList(dataSource.getId())); logger.log(Level.INFO, "Processing {0} blackboard artifacts.", listArtifacts.size()); //NON-NLS @@ -376,8 +364,7 @@ class DomainCategoryRunner extends Extract { for (BlackboardArtifact artifact : listArtifacts) { // make sure we haven't cancelled if (context.dataSourceIngestIsCancelled()) { - //User cancelled the process. - break; + break; //User cancelled the process. } // get the pertinent details for this artifact. From 980651a67b81c47268b70e7a9563fba4e91319aa Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 30 Mar 2021 12:03:59 -0400 Subject: [PATCH 04/15] updated properties-MERGED file --- .../url/analytics/domaincategorization/Bundle.properties-MERGED | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED index 68993c6213..f08c1c2986 100644 --- a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED @@ -27,7 +27,7 @@ AddEditCategoryDialog.categoryLabel.text=Category: AddEditCategoryDialog.domainSuffixLabel.text=Domain Suffix: AddEditCategoryDialog.saveButton.text=Save AddEditCategoryDialog.cancelButton.text=Cancel -WebCategoriesOptionsPanel.panelDescription.text=This module allows you to classify web sites based on domain names. +WebCategoriesOptionsPanel.panelDescription.text=This feature allows the Recent Activity ingest module to perform custom categorization of web sites based on domain suffixes. WebCategoriesOptionsPanel.categoriesTitle.text=Categories: WebCategoriesOptionsPanel.newEntryButton.text=New Entry WebCategoriesOptionsPanel.editEntryButton.text=Edit Entry From dd12f5bf6d0fe7062a79331d21356b73ac8d2c28 Mon Sep 17 00:00:00 2001 From: apriestman Date: Tue, 30 Mar 2021 13:49:03 -0400 Subject: [PATCH 05/15] Change column name from unique_id to addr in test script --- test/script/tskdbdiff.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index 9452b335d9..dcf75031ec 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -798,7 +798,7 @@ def build_id_accounts_table(db_cursor, isPostgreSQL): """ # for each row in the db, take the object id and account SID then creates a tuple in the dictionary # with the object id as the key and the OS Account's SID as the value - mapping = dict([(row[0], row[1]) for row in sql_select_execute(db_cursor, isPostgreSQL, "SELECT os_account_obj_id, unique_id FROM tsk_os_accounts")]) + mapping = dict([(row[0], row[1]) for row in sql_select_execute(db_cursor, isPostgreSQL, "SELECT os_account_obj_id, addr FROM tsk_os_accounts")]) return mapping def build_id_obj_path_table(files_table, objects_table, artifacts_table, reports_table, images_table, accounts_table): @@ -810,7 +810,7 @@ def build_id_obj_path_table(files_table, objects_table, artifacts_table, reports artifacts_table: obj_id, artifact_type_name reports_table: obj_id, path images_table: obj_id, name - accounts_table: obj_id, unique_id + accounts_table: obj_id, addr """ # make a copy of files_table and update it with new data from artifacts_table and reports_table mapping = files_table.copy() @@ -830,7 +830,7 @@ def build_id_obj_path_table(files_table, objects_table, artifacts_table, reports elif par_obj_id in images_table.keys(): path = images_table[par_obj_id] mapping[k] = path + "/" + artifacts_table[k] - elif k in accounts_table.keys(): # For an OS Account object ID we use its unique_id field which is the account SID + elif k in accounts_table.keys(): # For an OS Account object ID we use its addr field which is the account SID mapping[k] = accounts_table[k] elif v[0] not in mapping.keys(): if v[0] in artifacts_table.keys(): From 6f5b075f474d85453a586a451ec12e52ed966c2e Mon Sep 17 00:00:00 2001 From: apriestman Date: Tue, 30 Mar 2021 14:13:59 -0400 Subject: [PATCH 06/15] Fix icon capitalization --- Core/src/org/sleuthkit/autopsy/datamodel/utils/IconsUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/utils/IconsUtil.java b/Core/src/org/sleuthkit/autopsy/datamodel/utils/IconsUtil.java index 78cee32000..de88e41e04 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/utils/IconsUtil.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/utils/IconsUtil.java @@ -70,7 +70,7 @@ public final class IconsUtil { } else if (typeID == ARTIFACT_TYPE.TSK_SPEED_DIAL_ENTRY.getTypeID()) { imageFile = "speeddialentry.png"; //NON-NLS } else if (typeID == ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING.getTypeID()) { - imageFile = "bluetooth.png"; //NON-NLS + imageFile = "Bluetooth.png"; //NON-NLS } else if (typeID == ARTIFACT_TYPE.TSK_GPS_BOOKMARK.getTypeID()) { imageFile = "gpsfav.png"; //NON-NLS } else if (typeID == ARTIFACT_TYPE.TSK_GPS_LAST_KNOWN_LOCATION.getTypeID()) { From 996cd1f0bc76beed7367e8ab9d1c43dfac5dc48d Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Tue, 30 Mar 2021 16:38:09 -0400 Subject: [PATCH 07/15] Update OsAccountDataViewer --- .../osaccount/Bundle.properties-MERGED | 5 ++ .../osaccount/OsAccountDataPanel.java | 71 ++++++++++++++----- 2 files changed, 57 insertions(+), 19 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/Bundle.properties-MERGED index 4a962469c1..d74307642c 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/Bundle.properties-MERGED @@ -1,3 +1,4 @@ +OsAccountDataPanel_administrator_title=Administrator OsAccountDataPanel_basic_address=Address OsAccountDataPanel_basic_admin=Administrator OsAccountDataPanel_basic_creationDate=Creation Date @@ -5,6 +6,10 @@ OsAccountDataPanel_basic_fullname=Full Name OsAccountDataPanel_basic_login=Login OsAccountDataPanel_basic_title=Basic Properties OsAccountDataPanel_basic_type=Type +OsAccountDataPanel_data_accessed_title=Last Login +OsAccountDataPanel_host_count_title=Login Count +# {0} - hostName +OsAccountDataPanel_host_section_title={0} Details OsAccountDataPanel_realm_address=Address OsAccountDataPanel_realm_confidence=Confidence OsAccountDataPanel_realm_name=Name diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java index 79b22d81cb..2117b86cc6 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java @@ -41,6 +41,7 @@ import javax.swing.SwingWorker; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.contentviewers.osaccount.SectionData.RowData; +import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.OsAccount; @@ -48,6 +49,7 @@ import org.sleuthkit.datamodel.OsAccountAttribute; import org.sleuthkit.datamodel.OsAccountInstance; import org.sleuthkit.datamodel.OsAccountManager; import org.sleuthkit.datamodel.OsAccountRealm; +import org.sleuthkit.datamodel.SleuthkitCase; /** * Panel for displaying the properties of an OsAccount. @@ -82,7 +84,6 @@ public class OsAccountDataPanel extends JPanel { * @param account OsAccount to display, if null is passed the panel will * appear blank. */ -// void setOsAccount(OsAccount account) { void setOsAccountId(Long osAccountId) { removeAll(); revalidate(); @@ -225,10 +226,33 @@ public class OsAccountDataPanel extends JPanel { return data; } + @Messages({ + "# {0} - hostName", + "OsAccountDataPanel_host_section_title={0} Details", + "OsAccountDataPanel_host_count_title=Login Count", + "OsAccountDataPanel_data_accessed_title=Last Login", + "OsAccountDataPanel_administrator_title=Administrator" + }) private SectionData buildHostData(Host host, List attributeList) { - SectionData data = new SectionData(host.getName()); + SectionData data = new SectionData(Bundle.OsAccountDataPanel_host_section_title(host.getName())); for (OsAccountAttribute attribute : attributeList) { - data.addData(attribute.getAttributeType().getDisplayName(), attribute.getDisplayString()); + String displayName = attribute.getAttributeType().getDisplayName(); + String value = attribute.getDisplayString(); + + if(attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COUNT.getTypeID()) { + displayName = Bundle.OsAccountDataPanel_host_count_title(); + } else if(attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IS_ADMIN.getTypeID()) { + displayName = Bundle.OsAccountDataPanel_administrator_title(); + if(attribute.getValueInt() == 0) { + value = "False"; + } else { + value = "True"; + } + } else if(attribute.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID()) { + displayName = Bundle.OsAccountDataPanel_data_accessed_title(); + } + + data.addData(displayName, value); } return data; @@ -254,7 +278,7 @@ public class OsAccountDataPanel extends JPanel { * @param row The row in the layout. */ private void addPropertyName(String key, int row) { - JLabel label = new JLabel(key); + JLabel label = new JLabel(key + ":"); add(label, getPropertyNameContraints(row)); } @@ -359,7 +383,9 @@ public class OsAccountDataPanel extends JPanel { protected WorkerResults doInBackground() throws Exception { Map> hostMap = new HashMap<>(); Map instanceMap = new HashMap<>(); - OsAccountManager osAccountManager = Case.getCurrentCase().getSleuthkitCase().getOsAccountManager(); + SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); + OsAccountManager osAccountManager = skCase.getOsAccountManager(); + OsAccountRealm realm = skCase.getOsAccountRealmManager().getRealmById(account.getRealmId()); if(account == null) { account = osAccountManager.getOsAccountByObjectId(accountId); @@ -414,7 +440,7 @@ public class OsAccountDataPanel extends JPanel { } } - return new WorkerResults(hostMap, instanceMap); + return new WorkerResults(hostMap, instanceMap, realm); } @Override @@ -442,20 +468,21 @@ public class OsAccountDataPanel extends JPanel { hostDataMap.forEach((K, V) -> data.add(buildHostData(K, V))); } - // TODO - load realm on background thread - //OsAccountRealm realm = account.getRealm(); - //if (realm != null) { - // data.add(buildRealmProperties(realm)); - //} - - Map instanceMap = results.getDataSourceMap(); - if (!instanceMap.isEmpty()) { - SectionData instanceSection = new SectionData("Instances"); - instanceMap.forEach((K, V) -> instanceSection.addData(K.getName(), V.getName())); - - data.add(instanceSection); + OsAccountRealm realm = results.getRealm(); + if (realm != null) { + data.add(buildRealmProperties(realm)); } +// Removing the instance section for now. Leaving code here for +// future use. +// Map instanceMap = results.getDataSourceMap(); +// if (!instanceMap.isEmpty()) { +// SectionData instanceSection = new SectionData("Instances"); +// instanceMap.forEach((K, V) -> instanceSection.addData(K.getName(), V.getName())); +// +// data.add(instanceSection); +// } + addDataComponents(data); revalidate(); @@ -472,6 +499,7 @@ public class OsAccountDataPanel extends JPanel { private final Map> attributeMap; private final Map instanceMap; + private final OsAccountRealm realm; /** * Construct a new WorkerResult object. @@ -481,9 +509,10 @@ public class OsAccountDataPanel extends JPanel { * @param instanceMap A map of data to display OsAccount instance * information. */ - WorkerResults(Map> attributeMap, Map instanceMap) { + WorkerResults(Map> attributeMap, Map instanceMap, OsAccountRealm realm) { this.attributeMap = attributeMap; this.instanceMap = instanceMap; + this.realm = realm; } /** @@ -505,5 +534,9 @@ public class OsAccountDataPanel extends JPanel { Map getDataSourceMap() { return instanceMap; } + + OsAccountRealm getRealm() { + return realm; + } } } From a322b21830eaac3534b98f6b06c7a3e72b4f577d Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Mar 2021 17:56:49 -0400 Subject: [PATCH 08/15] 2021-03-29 normalize object ids for tsk_data_artifacts table --- test/script/tskdbdiff.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index dcf75031ec..cec54316d2 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -445,6 +445,7 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info os_account_index = line.find('INSERT INTO "tsk_os_accounts"') > -1 or line.find('INSERT INTO tsk_os_accounts') > -1 os_account_attr_index = line.find('INSERT INTO "tsk_os_account_attributes"') > -1 or line.find('INSERT INTO tsk_os_account_attributes') > -1 os_account_instances_index = line.find('INSERT INTO "tsk_os_account_instances"') > -1 or line.find('INSERT INTO tsk_os_account_instances') > -1 + data_artifacts_index = line.find('INSERT INTO "tsk_data_artifacts"') > -1 or line.find('INSERT INTO tsk_data_artifacts') > -1 parens = line[line.find('(') + 1 : line.rfind(')')] no_space_parens = parens.replace(" ", "") @@ -670,6 +671,19 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info fields_list[1] = accounts_table[os_account_id] newLine = ('INSERT INTO "tsk_os_account_instances" VALUES(' + ','.join(fields_list[1:]) + ');') # remove id return newLine + elif data_artifacts_index: + art_obj_id = int(fields_list[0]) + if art_obj_id in files_table.keys(): + fields_list[0] = files_table[art_obj_id] + else: + fields_list[0] = 'Artifact Object ID Omitted' + account_obj_id = int(fields_list[1]) + if account_obj_id in files_table.keys(): + fields_list[1] = files_table[account_obj_id] + else: + fields_list[1] = 'Account Object ID Omitted' + newLine = ('INSERT INTO "tsk_data_artifacts" VALUES(' + ','.join(fields_list[:]) + ');') # remove ids + return newLine else: return line From 172d465aeb79c436a8fda7d219033db84292b6d1 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 31 Mar 2021 11:57:02 -0400 Subject: [PATCH 09/15] 7360 Eliminate IngestManager/IngestJobPipeline deadlock --- .../sleuthkit/autopsy/ingest/IngestJob.java | 62 +++++++++++-------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index decccd4e84..9609724c64 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2018 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -63,7 +63,7 @@ public final class IngestJob { return displayName; } } - + /** * Ingest job mode. */ @@ -71,7 +71,7 @@ public final class IngestJob { BATCH, STREAMING } - + private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); private final static AtomicLong nextId = new AtomicLong(0L); private final long id; @@ -113,12 +113,12 @@ public final class IngestJob { this(Arrays.asList(dataSource), settings); this.files.addAll(files); } - + /** - * Constructs an ingest job that analyzes one data source, possibly using - * an ingest stream. + * Constructs an ingest job that analyzes one data source, possibly using an + * ingest stream. * - * @param settings The ingest job settings. + * @param settings The ingest job settings. */ IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) { this.id = IngestJob.nextId.getAndIncrement(); @@ -149,10 +149,10 @@ public final class IngestJob { boolean hasIngestPipeline() { return (!settings.getEnabledIngestModuleTemplates().isEmpty()); } - + /** * Add a set of files (by object ID) to be ingested. - * + * * @param fileObjIds the list of file IDs */ void addStreamingIngestFiles(List fileObjIds) { @@ -164,7 +164,7 @@ public final class IngestJob { IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); streamingIngestPipeline.addStreamingIngestFiles(fileObjIds); } - + /** * Start data source processing for streaming ingest. */ @@ -185,7 +185,7 @@ public final class IngestJob { * @return A collection of ingest module start up errors, empty on success. */ List start() { - + /* * Set up the pipeline(s) */ @@ -199,11 +199,11 @@ public final class IngestJob { this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); } incompleteJobsCount.set(ingestJobPipelines.size()); - + /* * Try to start each data source ingest job. Note that there is an - * assumption here that if there is going to be a module - * startup failure, it will be for the first ingest job pipeline. + * assumption here that if there is going to be a module startup + * failure, it will be for the first ingest job pipeline. * * TODO (RC): Consider separating module start up from pipeline startup * so that no processing is done if this assumption is false. @@ -229,14 +229,14 @@ public final class IngestJob { return errors; } - + /** * Get the ingest mode for this job (batch or streaming). - * + * * @return the ingest mode. */ Mode getIngestMode() { - return ingestMode; + return ingestMode; } /** @@ -251,8 +251,8 @@ public final class IngestJob { /** * Gets a snapshot of the progress of this ingest job. * - * @param getIngestTasksSnapshot - * + * @param getIngestTasksSnapshot + * * @return The snapshot. */ public ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot) { @@ -295,10 +295,21 @@ public final class IngestJob { * @param reason The reason for cancellation. */ public void cancel(CancellationReason reason) { - this.cancellationReason = reason; - this.ingestJobPipelines.values().stream().forEach((job) -> { - job.cancel(reason); - }); + cancellationReason = reason; + /* + * Cancel the ingest pipelines for each data source. This is done in a + * separate thread to avoid a potential deadlock. The deadlock is + * possible because this method can be called in a thread that acquires + * the ingest manager's ingest jobs list lock and then tries to acquire + * the ingest pipeline stage transition lock, while an ingest thread + * that has acquired the stage transition lock then tries to acquire the + * ingest manager's ingest jobs list lock. + */ + new Thread(() -> { + this.ingestJobPipelines.values().stream().forEach((job) -> { + job.cancel(reason); + }); + }).start(); } /** @@ -508,8 +519,9 @@ public final class IngestJob { * used to get basic information about the module and to request * cancellation of the module. * - * @param ingestJobPipeline The ingestJobPipeline that owns the data source level ingest module. - * @param module The data source level ingest module. + * @param ingestJobPipeline The ingestJobPipeline that owns the data + * source level ingest module. + * @param module The data source level ingest module. */ private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) { this.ingestJobPipeline = ingestJobPipeline; From d1a6e097c7e38c5d1c24b49f54fda480c6727d15 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 31 Mar 2021 12:00:33 -0400 Subject: [PATCH 10/15] Format IngestJob.java --- .../sleuthkit/autopsy/ingest/IngestJob.java | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index decccd4e84..6f6dc80b87 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -63,7 +63,7 @@ public final class IngestJob { return displayName; } } - + /** * Ingest job mode. */ @@ -71,7 +71,7 @@ public final class IngestJob { BATCH, STREAMING } - + private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); private final static AtomicLong nextId = new AtomicLong(0L); private final long id; @@ -113,12 +113,12 @@ public final class IngestJob { this(Arrays.asList(dataSource), settings); this.files.addAll(files); } - + /** - * Constructs an ingest job that analyzes one data source, possibly using - * an ingest stream. + * Constructs an ingest job that analyzes one data source, possibly using an + * ingest stream. * - * @param settings The ingest job settings. + * @param settings The ingest job settings. */ IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) { this.id = IngestJob.nextId.getAndIncrement(); @@ -149,10 +149,10 @@ public final class IngestJob { boolean hasIngestPipeline() { return (!settings.getEnabledIngestModuleTemplates().isEmpty()); } - + /** * Add a set of files (by object ID) to be ingested. - * + * * @param fileObjIds the list of file IDs */ void addStreamingIngestFiles(List fileObjIds) { @@ -164,7 +164,7 @@ public final class IngestJob { IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); streamingIngestPipeline.addStreamingIngestFiles(fileObjIds); } - + /** * Start data source processing for streaming ingest. */ @@ -185,7 +185,7 @@ public final class IngestJob { * @return A collection of ingest module start up errors, empty on success. */ List start() { - + /* * Set up the pipeline(s) */ @@ -199,11 +199,11 @@ public final class IngestJob { this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); } incompleteJobsCount.set(ingestJobPipelines.size()); - + /* * Try to start each data source ingest job. Note that there is an - * assumption here that if there is going to be a module - * startup failure, it will be for the first ingest job pipeline. + * assumption here that if there is going to be a module startup + * failure, it will be for the first ingest job pipeline. * * TODO (RC): Consider separating module start up from pipeline startup * so that no processing is done if this assumption is false. @@ -229,14 +229,14 @@ public final class IngestJob { return errors; } - + /** * Get the ingest mode for this job (batch or streaming). - * + * * @return the ingest mode. */ Mode getIngestMode() { - return ingestMode; + return ingestMode; } /** @@ -251,8 +251,8 @@ public final class IngestJob { /** * Gets a snapshot of the progress of this ingest job. * - * @param getIngestTasksSnapshot - * + * @param getIngestTasksSnapshot + * * @return The snapshot. */ public ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot) { @@ -508,8 +508,9 @@ public final class IngestJob { * used to get basic information about the module and to request * cancellation of the module. * - * @param ingestJobPipeline The ingestJobPipeline that owns the data source level ingest module. - * @param module The data source level ingest module. + * @param ingestJobPipeline The ingestJobPipeline that owns the data + * source level ingest module. + * @param module The data source level ingest module. */ private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) { this.ingestJobPipeline = ingestJobPipeline; From ba2c57e1dd3b24842a6c16a14e2c5f9b3b0fe8bc Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 31 Mar 2021 12:03:40 -0400 Subject: [PATCH 11/15] 7360 Eliminate IngestManager/IngestJobPipeline deadlock --- Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 9609724c64..76bbec2365 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -302,7 +302,7 @@ public final class IngestJob { * possible because this method can be called in a thread that acquires * the ingest manager's ingest jobs list lock and then tries to acquire * the ingest pipeline stage transition lock, while an ingest thread - * that has acquired the stage transition lock then tries to acquire the + * that has acquired the stage transition lock is trying to acquire the * ingest manager's ingest jobs list lock. */ new Thread(() -> { From 6299d862cc7e92687982ddfe77b4eda27e890d27 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 31 Mar 2021 12:21:04 -0400 Subject: [PATCH 12/15] 7090 Fix concurrent mod ex in IngestManager --- .../autopsy/ingest/IngestManager.java | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 4b3fe4e119..bb6a8991e0 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -125,7 +125,9 @@ public class IngestManager implements IngestProgressSnapshotProvider { private final int numberOfFileIngestThreads; private final AtomicLong nextIngestManagerTaskId = new AtomicLong(0L); private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-start-ingest-jobs-%d").build()); //NON-NLS; + @GuardedBy("startIngestJobFutures") private final Map> startIngestJobFutures = new ConcurrentHashMap<>(); + @GuardedBy("ingestJobsById") private final Map ingestJobsById = new HashMap<>(); private final ExecutorService dataSourceLevelIngestJobTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS; private final ExecutorService fileLevelIngestJobTasksExecutor; @@ -338,7 +340,9 @@ public class IngestManager implements IngestProgressSnapshotProvider { if (job.hasIngestPipeline()) { long taskId = nextIngestManagerTaskId.incrementAndGet(); Future task = startIngestJobsExecutor.submit(new StartIngestJobTask(taskId, job)); - startIngestJobFutures.put(taskId, task); + synchronized (startIngestJobFutures) { + startIngestJobFutures.put(taskId, task); + } } } } @@ -357,7 +361,9 @@ public class IngestManager implements IngestProgressSnapshotProvider { if (job.hasIngestPipeline()) { long taskId = nextIngestManagerTaskId.incrementAndGet(); Future task = startIngestJobsExecutor.submit(new StartIngestJobTask(taskId, job)); - startIngestJobFutures.put(taskId, task); + synchronized (startIngestJobFutures) { + startIngestJobFutures.put(taskId, task); + } } } } @@ -518,9 +524,11 @@ public class IngestManager implements IngestProgressSnapshotProvider { * @param reason The cancellation reason. */ public void cancelAllIngestJobs(IngestJob.CancellationReason reason) { - startIngestJobFutures.values().forEach((handle) -> { - handle.cancel(true); - }); + synchronized (startIngestJobFutures) { + startIngestJobFutures.values().forEach((handle) -> { + handle.cancel(true); + }); + } synchronized (ingestJobsById) { this.ingestJobsById.values().forEach((job) -> { job.cancel(reason); @@ -939,8 +947,10 @@ public class IngestManager implements IngestProgressSnapshotProvider { if (progress != null) { progress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.cancelling", displayName)); } - Future handle = startIngestJobFutures.remove(threadId); - handle.cancel(true); + synchronized (startIngestJobFutures) { + Future handle = startIngestJobFutures.remove(threadId); + handle.cancel(true); + } return true; } }); @@ -954,7 +964,9 @@ public class IngestManager implements IngestProgressSnapshotProvider { if (null != progress) { progress.finish(); } - startIngestJobFutures.remove(threadId); + synchronized (startIngestJobFutures) { + startIngestJobFutures.remove(threadId); + } } } From 6a9a878fb2b76e1f6422c15991be10e5b886006a Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 31 Mar 2021 15:46:14 -0400 Subject: [PATCH 13/15] Added supress deprecation annotation for tsk_os_account --- .../sleuthkit/autopsy/report/modules/html/HTMLReport.java | 1 + .../autopsy/report/modules/stix/EvalAccountObj.java | 8 ++++---- .../autopsy/recentactivity/ExtractRecycleBin.java | 2 -- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/html/HTMLReport.java b/Core/src/org/sleuthkit/autopsy/report/modules/html/HTMLReport.java index d1008f110b..aa084d66f3 100644 --- a/Core/src/org/sleuthkit/autopsy/report/modules/html/HTMLReport.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/html/HTMLReport.java @@ -213,6 +213,7 @@ public class HTMLReport implements TableReportModule { * Copies a suitable icon for the given data type in the output directory * and returns the icon file name to use for the given data type. */ + @SuppressWarnings( "deprecation" ) private String useDataTypeIcon(String dataType) { String iconFilePath; String iconFileName; diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/stix/EvalAccountObj.java b/Core/src/org/sleuthkit/autopsy/report/modules/stix/EvalAccountObj.java index 5d58f10388..e52d28d943 100644 --- a/Core/src/org/sleuthkit/autopsy/report/modules/stix/EvalAccountObj.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/stix/EvalAccountObj.java @@ -37,14 +37,14 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; */ class EvalAccountObj extends EvaluatableObject { - private AccountObjectType obj; + private final AccountObjectType obj; - public EvalAccountObj(AccountObjectType a_obj, String a_id, String a_spacing) { + EvalAccountObj(AccountObjectType a_obj, String a_id, String a_spacing) { obj = a_obj; id = a_id; spacing = a_spacing; } - + @SuppressWarnings( "deprecation" ) @Override public synchronized ObservableResult evaluate() { @@ -103,7 +103,7 @@ class EvalAccountObj extends EvaluatableObject { // The assumption here is that there aren't going to be too many network shares, so we // can cycle through all of them. try { - List finalHits = new ArrayList(); + List finalHits = new ArrayList<>(); Case case1 = Case.getCurrentCaseThrows(); SleuthkitCase sleuthkitCase = case1.getSleuthkitCase(); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java index 8e6ad2df8e..4478f49278 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java @@ -48,11 +48,9 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Blackboard.BlackboardException; import org.sleuthkit.datamodel.BlackboardArtifact; -import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_OS_ACCOUNT; import org.sleuthkit.datamodel.BlackboardAttribute; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_DELETED; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH; -import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_ID; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_NAME; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; From 0a01daa39441bfae36345440f4386d154a8be265 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 1 Apr 2021 08:04:57 -0400 Subject: [PATCH 14/15] fix for os account attribute api changes --- .../sleuthkit/autopsy/recentactivity/ExtractRegistry.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 1699042b52..0c5cd988db 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -1986,7 +1986,7 @@ class ExtractRegistry extends Extract { String dir = homeDir.replaceFirst("^(%\\w*%)", ""); dir = dir.replace("\\", "/"); attributes.add(createOsAccountAttribute(TSK_HOME_DIR, dir, osAccount, host, file)); - osAccount.addAttributes(attributes); + accountMgr.addOsAccountAttributes(osAccount, attributes); } accountMgr.updateOsAccount(osAccount); @@ -2163,8 +2163,9 @@ class ExtractRegistry extends Extract { groups, osAccount, host, regFile)); } - osAccount.addAttributes(attributes); - tskCase.getOsAccountManager().updateOsAccount(osAccount); + OsAccountManager accountMgr = tskCase.getOsAccountManager(); + accountMgr.addOsAccountAttributes(osAccount, attributes); + accountMgr.updateOsAccount(osAccount); } /** From 2c46a330669a5b1bba84d0423a2cb0754a60ed88 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 1 Apr 2021 09:58:12 -0400 Subject: [PATCH 15/15] Format IngestTaskPipeline.java --- Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java index 2866987f46..194cbec4e1 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java @@ -322,11 +322,11 @@ abstract class IngestTaskPipeline { * performing the task. */ abstract void performTask(IngestJobPipeline ingestJobPipeline, T task) throws IngestModuleException; - + @Override public void shutDown() { module.shutDown(); - } + } }