From 3ca95fb523f0860db742d41faf0737eb8482bc69 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 11 Mar 2021 15:08:45 -0500 Subject: [PATCH 01/17] Removed TSK_OS_ACCOUNT from RA --- .../recentactivity/ExtractRecycleBin.java | 21 +- .../recentactivity/ExtractRegistry.java | 276 ++---------------- 2 files changed, 24 insertions(+), 273 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java index e5b443fc1b..39e446c95f 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java @@ -35,6 +35,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import org.joda.time.Instant; import org.openide.util.NbBundle.Messages; @@ -54,7 +55,9 @@ import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PAT import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_ID; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_NAME; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.FsContent; +import org.sleuthkit.datamodel.OsAccount; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -336,21 +339,11 @@ final class ExtractRecycleBin extends Extract { */ private Map makeUserNameMap(Content dataSource) throws TskCoreException { Map userNameMap = new HashMap<>(); - - List accounts = blackboard.getArtifacts(TSK_OS_ACCOUNT.getTypeID(), dataSource.getId()); - - for (BlackboardArtifact account : accounts) { - BlackboardAttribute nameAttribute = getAttributeForArtifact(account, TSK_USER_NAME); - BlackboardAttribute idAttribute = getAttributeForArtifact(account, TSK_USER_ID); - - String userName = nameAttribute != null ? nameAttribute.getDisplayString() : ""; - String userID = idAttribute != null ? idAttribute.getDisplayString() : ""; - - if (!userID.isEmpty()) { - userNameMap.put(userID, userName); - } + + for(OsAccount account: tskCase.getOsAccountManager().getAccounts(((DataSource)dataSource).getHost())) { + Optional userName = account.getLoginName(); + userNameMap.put(account.getName(), userName.isPresent() ? userName.get() : ""); } - return userNameMap; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 8d215ccf6e..9ead2f3711 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -82,7 +82,6 @@ import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.Blackboard.BlackboardException; import org.sleuthkit.datamodel.BlackboardArtifact; import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT; -import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_OS_ACCOUNT; import org.sleuthkit.datamodel.BlackboardAttribute; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT; @@ -93,8 +92,6 @@ import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DAT import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH; -import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_ID; -import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_NAME; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HOME_DIR; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; @@ -866,65 +863,15 @@ class ExtractRegistry extends Extract { break; case "ProfileList": //NON-NLS - try { - String homeDir = value; - String sid = artnode.getAttribute("sid"); //NON-NLS - String username = artnode.getAttribute("username"); //NON-NLS - - // For now both an OsAccount and the - // TSK_OS_ACCOUNT artifact will be created. - try{ - createOrUpdateOsAccount(regFile, sid, username, homeDir); - - } catch(TskCoreException | TskDataException ex) { - logger.log(Level.SEVERE, String.format("Failed to create OsAccount for file: %s, sid: %s", regFile.getId(), sid)); - } - - BlackboardArtifact bbart = null; - try { - //check if any of the existing artifacts match this username - ArrayList existingArtifacts = currentCase.getSleuthkitCase().getBlackboardArtifacts(ARTIFACT_TYPE.TSK_OS_ACCOUNT); - for (BlackboardArtifact artifact : existingArtifacts) { - if (artifact.getDataSource().getId() == regFile.getDataSourceObjectId()) { - BlackboardAttribute attribute = artifact.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_USER_ID)); - if (attribute != null && attribute.getValueString().equals(sid)) { - bbart = artifact; - break; - } - } - } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error getting existing os account artifact", ex); - } - if (bbart == null) { - //create new artifact - bbart = regFile.newArtifact(ARTIFACT_TYPE.TSK_OS_ACCOUNT); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME, - parentModuleName, username)); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_ID, - parentModuleName, sid)); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH, - parentModuleName, homeDir)); + String homeDir = value; + String sid = artnode.getAttribute("sid"); //NON-NLS + String username = artnode.getAttribute("username"); //NON-NLS - newArtifacts.add(bbart); - } else { - //add attributes to existing artifact - BlackboardAttribute bbattr = bbart.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_USER_NAME)); + try{ + createOrUpdateOsAccount(regFile, sid, username, homeDir); - if (bbattr == null) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME, - parentModuleName, username)); - } - bbattr = bbart.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_PATH)); - if (bbattr == null) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH, - parentModuleName, homeDir)); - } - } - bbart.addAttributes(bbattributes); - - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error adding account artifact to blackboard.", ex); //NON-NLS + } catch(TskCoreException | TskDataException ex) { + logger.log(Level.SEVERE, String.format("Failed to create OsAccount for file: %s, sid: %s", regFile.getId(), sid), ex); } break; @@ -1171,32 +1118,6 @@ class ExtractRegistry extends Extract { updateOsAccount(osAccount, userInfo, groupMap.get(userInfo.get(SID_KEY)), regAbstractFile); } - // Existing TSK_OS_ACCOUNT code. - - //get all existing OS account artifacts - List existingOsAccounts = tskCase.getBlackboardArtifacts(ARTIFACT_TYPE.TSK_OS_ACCOUNT); - for (BlackboardArtifact osAccount : existingOsAccounts) { - //if the OS Account artifact was from the same data source check the user id - if (osAccount.getDataSource().getId() == regAbstractFile.getDataSourceObjectId()) { - BlackboardAttribute existingUserId = osAccount.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_USER_ID)); - if (existingUserId != null) { - String userID = existingUserId.getValueString().trim(); - Map userInfo = userInfoMap.remove(userID); - //if the existing user id matches a user id which we parsed information for check if that information exists and if it doesn't add it - if (userInfo != null) { - osAccount.addAttributes(getAttributesForAccount(userInfo, groupMap.get(userID), true, regAbstractFile)); - } - } - } - } - - //add remaining userinfos as accounts; - for (Map userInfo : userInfoMap.values()) { - BlackboardArtifact bbart = regAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_OS_ACCOUNT); - bbart.addAttributes(getAttributesForAccount(userInfo, groupMap.get(userInfo.get(SID_KEY)), false, regAbstractFile)); - // index the artifact for keyword search - newArtifacts.add(bbart); - } // Get a mapping of user sids to user names and save globally so it can be used for other areas // of the registry, ie: BAM key try { @@ -1212,8 +1133,6 @@ class ExtractRegistry extends Extract { logger.log(Level.WARNING, "Error finding the registry file.", ex); //NON-NLS } catch (IOException ex) { logger.log(Level.WARNING, "Error building the document parser: {0}", ex); //NON-NLS - } catch (ParseException ex) { - logger.log(Level.WARNING, "Error parsing the the date from the registry file", ex); //NON-NLS } catch (TskDataException | TskCoreException ex) { logger.log(Level.WARNING, "Error updating TSK_OS_ACCOUNT artifacts to include newly parsed data.", ex); //NON-NLS } finally { @@ -1223,163 +1142,6 @@ class ExtractRegistry extends Extract { } return false; } - - /** - * Creates the attribute list for the given user information and group list. - * - * @param userInfo Map of key\value pairs of user information - * @param groupList List of the groups that user belongs - * @param existingUser - * - * @return List - * - * @throws ParseException - */ - Collection getAttributesForAccount(Map userInfo, List groupList, boolean existingUser, AbstractFile regAbstractFile) throws ParseException { - Collection bbattributes = new ArrayList<>(); - - SimpleDateFormat regRipperTimeFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy 'Z'", US); - regRipperTimeFormat.setTimeZone(getTimeZone("GMT")); - - if (!existingUser) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_ID, - getRAModuleName(), userInfo.get(SID_KEY))); - - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME, - this.moduleName, userInfo.get(USERNAME_KEY))); - } - - String value = userInfo.get(ACCOUNT_CREATED_KEY); - if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, - getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); - } - - value = userInfo.get(LAST_LOGIN_KEY); - if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, - getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); - } - - value = userInfo.get(LOGIN_COUNT_KEY); - if (value != null && !value.isEmpty()) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, - getRAModuleName(), Integer.parseInt(value))); - } - - value = userInfo.get(ACCOUNT_TYPE_KEY); - if (value != null && !value.isEmpty()) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE, - getRAModuleName(), value)); - } - - value = userInfo.get(USER_COMMENT_KEY); - if (value != null && !value.isEmpty()) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DESCRIPTION, - getRAModuleName(), value)); - } - - value = userInfo.get(NAME_KEY); - if (value != null && !value.isEmpty()) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, - getRAModuleName(), value)); - } - - value = userInfo.get(INTERNET_NAME_KEY); - if (value != null && !value.isEmpty()) { - try { - // Create an account for this email, if it doesn't already exist. - Case.getCurrentCaseThrows() - .getSleuthkitCase() - .getCommunicationsManager() - .createAccountFileInstance(Account.Type.EMAIL, - value, getRAModuleName(), regAbstractFile); - } catch (NoCurrentCaseException | TskCoreException ex) { - logger.log(Level.SEVERE, - String.format("Error adding email account with value " - + "%s, to the case database for file %s [objId=%d]", - value, regAbstractFile.getName(), regAbstractFile.getId()), ex); - } - - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL, - getRAModuleName(), value)); - } - - value = userInfo.get(FULL_NAME_KEY); - if (value != null && !value.isEmpty()) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DISPLAY_NAME, - getRAModuleName(), value)); - } - - value = userInfo.get(PWD_RESET_KEY); - if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_PASSWORD_RESET, - getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); - } - - value = userInfo.get(PASSWORD_HINT); - if (value != null && !value.isEmpty()) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PASSWORD_HINT, - getRAModuleName(), value)); - } - - value = userInfo.get(PWD_FAILE_KEY); - if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_PASSWORD_FAIL, - getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); - } - - String settingString = ""; - for (String setting : PASSWORD_SETTINGS_FLAGS) { - if (userInfo.containsKey(setting)) { - settingString += setting + ", "; - } - } - - if (!settingString.isEmpty()) { - settingString = settingString.substring(0, settingString.length() - 2); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PASSWORD_SETTINGS, - getRAModuleName(), settingString)); - } - - settingString = ""; - for (String setting : ACCOUNT_SETTINGS_FLAGS) { - if (userInfo.containsKey(setting)) { - settingString += setting + ", "; - } - } - - if (!settingString.isEmpty()) { - settingString = settingString.substring(0, settingString.length() - 2); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ACCOUNT_SETTINGS, - getRAModuleName(), settingString)); - } - - settingString = ""; - for (String setting : ACCOUNT_TYPE_FLAGS) { - if (userInfo.containsKey(setting)) { - settingString += setting + ", "; - } - } - - if (!settingString.isEmpty()) { - settingString = settingString.substring(0, settingString.length() - 2); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_FLAG, - getRAModuleName(), settingString)); - } - - if (groupList != null && groupList.isEmpty()) { - String groups = ""; - for (String group : groupList) { - groups += group + ", "; - } - - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GROUPS, - getRAModuleName(), groups.substring(0, groups.length() - 2))); - } - - return bbattributes; - } /** * Read the User Information section of the SAM regripper plugin's output @@ -1949,23 +1711,14 @@ class ExtractRegistry extends Extract { * @throws TskCoreException */ private Map makeUserNameMap(Content dataSource) throws TskCoreException { - Map userNameMap = new HashMap<>(); + Map map = new HashMap<>(); - List accounts = blackboard.getArtifacts(TSK_OS_ACCOUNT.getTypeID(), dataSource.getId()); - - for (BlackboardArtifact account : accounts) { - BlackboardAttribute nameAttribute = getAttributeForArtifact(account, TSK_USER_NAME); - BlackboardAttribute idAttribute = getAttributeForArtifact(account, TSK_USER_ID); - - String userName = nameAttribute != null ? nameAttribute.getDisplayString() : ""; - String userID = idAttribute != null ? idAttribute.getDisplayString() : ""; - - if (!userID.isEmpty()) { - userNameMap.put(userID, userName); - } + for(OsAccount account: tskCase.getOsAccountManager().getAccounts(((DataSource)dataSource).getHost())) { + Optional userName = account.getLoginName(); + map.put(account.getName(), userName.isPresent() ? userName.get() : ""); } - return userNameMap; + return map; } /** @@ -2306,6 +2059,11 @@ class ExtractRegistry extends Extract { osAccount, host, regFile)); } } + + value = userInfo.get(USERNAME_KEY); + if (value != null && !value.isEmpty()) { + osAccount.setLoginName(value); + } value = userInfo.get(LOGIN_COUNT_KEY); if (value != null && !value.isEmpty()) { From 32c94623bfdb519da8920c7be14d5cc6cc083c06 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 19 Mar 2021 11:19:45 -0400 Subject: [PATCH 02/17] 7415 file pipeline file save batching --- .../src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 3ba603a968..2e242d6bc7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.ingest; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.sleuthkit.datamodel.AbstractFile; @@ -31,6 +32,8 @@ final class FileIngestPipeline extends IngestTaskPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final IngestJobPipeline ingestJobPipeline; + private final Object fileBatchLock; + private final List fileBatch; /** * Constructs a pipeline of file ingest modules for performing file ingest @@ -43,6 +46,8 @@ final class FileIngestPipeline extends IngestTaskPipeline { FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { super(ingestJobPipeline, moduleTemplates); this.ingestJobPipeline = ingestJobPipeline; + fileBatchLock = new Object(); + fileBatch = new ArrayList<>(); } @Override From 1bbeb5fac69daec3f081e7d5312d0fcac1295c7f Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Mar 2021 16:28:43 -0400 Subject: [PATCH 03/17] 7415 batch file saves during ingest --- .../autopsy/ingest/FileIngestPipeline.java | 71 +++++++++++++------ .../autopsy/ingest/IngestManager.java | 2 +- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index fc68d48a57..2d54b22b7e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Optional; import org.sleuthkit.datamodel.AbstractFile; @@ -30,9 +31,9 @@ import org.sleuthkit.datamodel.TskCoreException; */ final class FileIngestPipeline extends IngestTaskPipeline { + private static final int FILE_BATCH_SIZE = 500; private static final IngestManager ingestManager = IngestManager.getInstance(); private final IngestJobPipeline ingestJobPipeline; - private final Object fileBatchLock; private final List fileBatch; /** @@ -46,7 +47,6 @@ final class FileIngestPipeline extends IngestTaskPipeline { FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { super(ingestJobPipeline, moduleTemplates); this.ingestJobPipeline = ingestJobPipeline; - fileBatchLock = new Object(); fileBatch = new ArrayList<>(); } @@ -66,32 +66,61 @@ final class FileIngestPipeline extends IngestTaskPipeline { @Override void completeTask(FileIngestTask task) throws IngestTaskPipelineException { - ingestManager.setIngestTaskProgress(task, "Saving Files"); //NON-NLS - AbstractFile file = null; - try { - file = task.getFile(); - } catch (TskCoreException ex) { - throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS - } - try { - if (!ingestJobPipeline.isCancelled()) { - /* - * Save any updates from the ingest modules to the case - * database. - */ - file.save(); + ingestManager.setIngestTaskProgress(task, "Saving Files"); //NON-NLS // RJCTODO + synchronized (fileBatch) { + AbstractFile file = null; + try { + file = task.getFile(); + file.close(); + } catch (TskCoreException ex) { + // RJCTODO: Is this right? + throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS } - } catch (TskCoreException ex) { - throw new IngestTaskPipelineException(String.format("Failed to save updated data for file (file objId = %d)", task.getFileId()), ex); //NON-NLS - } finally { if (!ingestJobPipeline.isCancelled()) { - IngestManager.getInstance().fireFileIngestDone(file); + fileBatch.add(file); + if (fileBatch.size() >= FILE_BATCH_SIZE) { + saveFiles(); + } } - file.close(); ingestManager.setIngestTaskProgressCompleted(task); } } + @Override + List shutDown() { + Date start = new Date(); + saveFiles(); + Date finish = new Date(); + ingestManager.incrementModuleRunTime("Save Files", finish.getTime() - start.getTime()); // RJCTODO + return super.shutDown(); + } + + private void saveFiles() { + synchronized (fileBatch) { + for (AbstractFile file : fileBatch) { + try { + if (!ingestJobPipeline.isCancelled()) { + /* + * Save any updates from the ingest modules to the case + * database. + */ + file.save(); + } + } catch (TskCoreException ex) { + // RJCTODO: Log instead? +// throw new IngestTaskPipelineException(String.format("Failed to save updated data for file (file objId = %d)", task.getFileId()), ex); //NON-NLS + } finally { + if (!ingestJobPipeline.isCancelled()) { + IngestManager.getInstance().fireFileIngestDone(file); + } + file.close(); + //ingestManager.setIngestTaskProgressCompleted(task); + } + } + fileBatch.clear(); + } + } + /** * A wrapper that adds ingest infrastructure operations to a file ingest * module. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index f73960aeca..4b3fe4e119 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -837,7 +837,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { * @param moduleDisplayName The diplay name of the ingest module. * @param duration */ - private void incrementModuleRunTime(String moduleDisplayName, Long duration) { + void incrementModuleRunTime(String moduleDisplayName, Long duration) { if (moduleDisplayName.equals("IDLE")) { //NON-NLS return; } From 258ab2e74a165ad38a7681bcdd0bc429bc52c80a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 25 Mar 2021 10:08:03 -0400 Subject: [PATCH 04/17] 7415 file pipeline file save batching --- .../ingest/DataSourceIngestPipeline.java | 1 + .../autopsy/ingest/FileIngestPipeline.java | 89 ++++++++++++------- .../autopsy/ingest/IngestTaskPipeline.java | 1 - 3 files changed, 57 insertions(+), 34 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 3bef43e0b2..f48fc337f0 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -95,6 +95,7 @@ final class DataSourceIngestPipeline extends IngestTaskPipeline { void completeTask(FileIngestTask task) throws IngestTaskPipelineException { ingestManager.setIngestTaskProgress(task, Bundle.FileIngestPipeline_SaveResults_Activity()); /* - * Code in only one file ingest thread at a time will try to access the - * file list. The synchronization here is to ensure visibility of the - * files in all of the threads that share the list, rather than to - * prevent simultaneous access in multiple threads. + * Close and cache the file from the file ingest task. The cache will be + * used for an eventual batch update of the case database with new + * properties added to the files in the cache by the ingest modules that + * processed them. + * + * Only one file ingest thread at a time will try to access the file + * cache. The synchronization here is to ensure visibility of the files + * in all of the threads that share the cache, rather than to prevent + * simultaneous access in multiple threads. */ synchronized (fileBatch) { AbstractFile file = null; @@ -85,13 +92,12 @@ final class FileIngestPipeline extends IngestTaskPipeline { file = task.getFile(); file.close(); } catch (TskCoreException ex) { - // RJCTODO: Is this right? throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS } if (!ingestJobPipeline.isCancelled()) { fileBatch.add(file); if (fileBatch.size() >= FILE_BATCH_SIZE) { - updateFiles(); + clearFileCache(); } } ingestManager.setIngestTaskProgressCompleted(task); @@ -100,47 +106,63 @@ final class FileIngestPipeline extends IngestTaskPipeline { @Override List shutDown() { - Date start = new Date(); - updateFiles(); - Date finish = new Date(); - ingestManager.incrementModuleRunTime("Save Files", finish.getTime() - start.getTime()); // RJCTODO - return super.shutDown(); + List errors = new ArrayList<>(); + if (!ingestJobPipeline.isCancelled()) { + Date start = new Date(); + try { + clearFileCache(); + } catch (IngestTaskPipelineException ex) { + errors.add(new IngestModuleError(Bundle.FileIngestPipeline_SaveResults_Activity(), ex)); + } + Date finish = new Date(); + ingestManager.incrementModuleRunTime(Bundle.FileIngestPipeline_SaveResults_Activity(), finish.getTime() - start.getTime()); + } + errors.addAll(super.shutDown()); + return errors; } /** - * RJCTODO + * Updates the case database with new properties added to the files in the + * cache by the ingest modules that processed them. * - * @throws TskCoreException + * @throws IngestTaskPipelineException Exception thrown if the case database + * update fails. */ - private void updateFiles() throws TskCoreException { - Case currentCase = Case.getCurrentCase(); - SleuthkitCase caseDb = currentCase.getSleuthkitCase(); - SleuthkitCase.CaseDbTransaction transaction = caseDb.beginTransaction(); -// transaction.commit(); - + private void clearFileCache() throws IngestTaskPipelineException { + /* + * Only one file ingest thread at a time will try to access the file + * cache. The synchronization here is to ensure visibility of the files + * in all of the threads that share the cache, rather than to prevent + * simultaneous access in multiple threads. + */ synchronized (fileBatch) { - for (AbstractFile file : fileBatch) { - try { + CaseDbTransaction transaction = null; + try { + Case currentCase = Case.getCurrentCaseThrows(); + SleuthkitCase caseDb = currentCase.getSleuthkitCase(); + transaction = caseDb.beginTransaction(); + for (AbstractFile file : fileBatch) { if (!ingestJobPipeline.isCancelled()) { file.save(transaction); } - } catch (TskCoreException ex) { - // RJCTODO: Log instead? -// throw new IngestTaskPipelineException(String.format("Failed to save updated data for file (file objId = %d)", task.getFileId()), ex); //NON-NLS - } finally { - if (!ingestJobPipeline.isCancelled()) { + } + transaction.commit(); + if (!ingestJobPipeline.isCancelled()) { + for (AbstractFile file : fileBatch) { IngestManager.getInstance().fireFileIngestDone(file); } - file.close(); - //ingestManager.setIngestTaskProgressCompleted(task); } - } - for (AbstractFile file : fileBatch) { - if (!ingestJobPipeline.isCancelled()) { - IngestManager.getInstance().fireFileIngestDone(file); + } catch (NoCurrentCaseException | TskCoreException ex) { + if (transaction != null) { + try { + transaction.rollback(); + } catch (TskCoreException ignored) { + } } + throw new IngestTaskPipelineException("Failed to save updated properties for cached files from tasks", ex); //NON-NLS + } finally { + fileBatch.clear(); } - fileBatch.clear(); } } @@ -176,6 +198,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { ingestManager.setIngestTaskProgress(task, getDisplayName()); ingestJobPipeline.setCurrentFileIngestModule(getDisplayName(), file.getName()); ProcessResult result = module.process(file); + // See JIRA-7449 // if (result == ProcessResult.ERROR) { // throw new IngestModuleException(String.format("%s experienced an error analyzing %s (file objId = %d)", getDisplayName(), file.getName(), file.getId())); //NON-NLS // } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java index f7451dd9d8..2866987f46 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java @@ -36,7 +36,6 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; */ abstract class IngestTaskPipeline { - private static final IngestManager ingestManager = IngestManager.getInstance(); private final IngestJobPipeline ingestJobPipeline; private final List moduleTemplates; private final List> modules; From bef9e117e2c448c7061c98be3fc37fa81dc81f3c Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 25 Mar 2021 17:40:05 -0400 Subject: [PATCH 05/17] Fixed content panel and osAccount --- .../osaccount/OsAccountDataPanel.java | 32 +++++++++++++++- .../osaccount/OsAccountViewer.java | 37 ++++++++++--------- .../recentactivity/ChromeCacheExtractor.java | 8 +++- .../autopsy/recentactivity/Extract.java | 6 ++- .../recentactivity/RAOsAccountCache.java | 7 ++-- 5 files changed, 66 insertions(+), 24 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java index a1077e3e82..2a78256565 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java @@ -82,6 +82,24 @@ public class OsAccountDataPanel extends JPanel { * @param account OsAccount to display, if null is passed the panel will * appear blank. */ +// void setOsAccount(OsAccount account) { + void setOsAccountId(Long osAccountId) { + removeAll(); + revalidate(); + + if (osAccountId != null) { + setLayout(new BorderLayout()); + add(new JLabel("Loading OsAccount Data..."), BorderLayout.NORTH); + + if (dataFetcher != null && !dataFetcher.isDone()) { + dataFetcher.cancel(true); + } + + dataFetcher = new PanelDataFetcher(osAccountId); + dataFetcher.execute(); + } + } + void setOsAccount(OsAccount account) { removeAll(); revalidate(); @@ -319,15 +337,22 @@ public class OsAccountDataPanel extends JPanel { */ private class PanelDataFetcher extends SwingWorker { - private final OsAccount account; + private final Long accountId; + private OsAccount account; /** * Construct a new worker for the given account. * * @param account */ + PanelDataFetcher(Long accountId) { + this.accountId = accountId; + this.account = null; + } + PanelDataFetcher(OsAccount account) { this.account = account; + this.accountId = null; } @Override @@ -335,6 +360,11 @@ public class OsAccountDataPanel extends JPanel { Map> hostMap = new HashMap<>(); Map instanceMap = new HashMap<>(); OsAccountManager osAccountManager = Case.getCurrentCase().getSleuthkitCase().getOsAccountManager(); + + if(account == null) { + account = osAccountManager.getOsAccount(accountId); + } + List hosts = osAccountManager.getHosts(account); List attributeList = account.getOsAccountAttributes(); diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java index a56e4de8c3..dd43d1f531 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java @@ -53,26 +53,29 @@ public class OsAccountViewer extends javax.swing.JPanel implements DataContentVi @Override public void setNode(Node node) { - OsAccount osAccount = null; + Long osAccountId = null; try { - osAccount = node.getLookup().lookup(OsAccount.class); - if (osAccount == null) { - Optional optional; - AbstractFile file = node.getLookup().lookup(AbstractFile.class); - if (file != null) { - optional = file.getOsAccount(); - if (optional.isPresent()) { - osAccount = optional.get(); - } + OsAccount osAccount = node.getLookup().lookup(OsAccount.class); + if (osAccount != null) { + dataPanel.setOsAccount(osAccount); + return; + } + + Optional optional; + AbstractFile file = node.getLookup().lookup(AbstractFile.class); + if (file != null) { + optional = file.getOsAccountObjectId(); + if (optional.isPresent()) { + osAccountId = optional.get(); } } - if (osAccount == null) { + if (osAccountId == null) { DataArtifact dataArtifact = node.getLookup().lookup(DataArtifact.class); if (dataArtifact != null) { - Optional optional = dataArtifact.getOsAccount(); + optional = dataArtifact.getOsAccountObjectId(); if (optional.isPresent()) { - osAccount = optional.get(); + osAccountId = optional.get(); } } @@ -81,8 +84,8 @@ public class OsAccountViewer extends javax.swing.JPanel implements DataContentVi logger.log(Level.SEVERE, String.format("Failed to get OsAccount for node %s", node.getDisplayName()), ex); } - if (osAccount != null) { - dataPanel.setOsAccount(osAccount); + if (osAccountId != null) { + dataPanel.setOsAccountId(osAccountId); } } @@ -125,8 +128,8 @@ public class OsAccountViewer extends javax.swing.JPanel implements DataContentVi try { return osAccount != null - || (file != null && file.getOsAccount().isPresent()) - || (dataArtifact != null && dataArtifact.getOsAccount().isPresent()); + || (file != null && file.getOsAccountObjectId().isPresent()) + || (dataArtifact != null && dataArtifact.getOsAccountObjectId().isPresent()); } catch (TskCoreException ex) { logger.log(Level.SEVERE, String.format("Failed to determine if node %s is Supported for OsAccountViewer", node.getDisplayName()), ex); return false; diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java index 8b13eda7d1..ce60d94d7a 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java @@ -538,8 +538,12 @@ final class ChromeCacheExtractor { webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID, moduleName, cachedItemFile.getId())); - Optional optional = cacheEntryFile.getOsAccount(); - BlackboardArtifact webCacheArtifact = cacheEntryFile.newDataArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_CACHE), webAttr, optional.isPresent() ? optional.get() : null); + Optional optional = cacheEntryFile.getOsAccountObjectId(); + OsAccount account = null; + if(optional.isPresent()) { + account = currentCase.getSleuthkitCase().getOsAccountManager().getOsAccount(optional.get()); + } + BlackboardArtifact webCacheArtifact = cacheEntryFile.newDataArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_CACHE), webAttr, account); artifactsAdded.add(webCacheArtifact); // Create a TSK_ASSOCIATED_OBJECT on the f_XXX or derived file file back to the CACHE entry diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index 97c17b58dc..d7b4bb20a2 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -533,7 +533,11 @@ abstract class Extract { Optional getOsAccount(Content content) throws TskCoreException { if(content instanceof AbstractFile) { if(osAccountCache == null) { - return ((AbstractFile)content).getOsAccount(); + Optional accountId = ((AbstractFile)content).getOsAccountObjectId(); + if(accountId.isPresent()) { + return Optional.ofNullable(tskCase.getOsAccountManager().getOsAccount(accountId.get())); + } + return Optional.empty(); } return osAccountCache.getOsAccount(((AbstractFile)content)); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java index f5208807af..54a34196ed 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Host; @@ -71,18 +72,18 @@ final class RAOsAccountCache { * @throws TskCoreException */ Optional getOsAccount(AbstractFile file) throws TskCoreException { - Optional optional = file.getOsAccount(); + Optional optional = file.getOsAccountObjectId(); if (!optional.isPresent()) { return getAccountForPath(file.getParentPath()); } - OsAccount osAccount = optional.get(); + OsAccount osAccount = Case.getCurrentCase().getSleuthkitCase().getOsAccountManager().getOsAccount(optional.get()); if (osAccount.getName().equals("S-1-5-32-544")) { return getAccountForPath(file.getParentPath()); } - return optional; + return Optional.ofNullable(osAccount); } /** From 104d623354d40c22d2a7169b304a7df0b16794aa Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 25 Mar 2021 18:16:27 -0400 Subject: [PATCH 06/17] Modified OSAccountRemoveEvent to take id instead of object --- Core/src/org/sleuthkit/autopsy/casemodule/Case.java | 8 ++++---- .../casemodule/events/OsAccountRemovedEvent.java | 11 +++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index e5983f24d3..c0497c083c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -528,8 +528,8 @@ public class Case { @Subscribe public void publishOsAccountDeletedEvent(OsAccountsDeleteEvent event) { - for(OsAccount account: event.getOsAcounts()) { - eventPublisher.publish(new OsAccountRemovedEvent(account)); + for(Long accountId: event.getOsAcountObjectIds()) { + eventPublisher.publish(new OsAccountRemovedEvent(accountId)); } } @@ -1802,8 +1802,8 @@ public class Case { eventPublisher.publish(new OsAccountChangedEvent(account)); } - public void notifyOsAccountRemoved(OsAccount account) { - eventPublisher.publish(new OsAccountRemovedEvent(account)); + public void notifyOsAccountRemoved(Long osAccountObjectId) { + eventPublisher.publish(new OsAccountRemovedEvent(osAccountObjectId)); } /** diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountRemovedEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountRemovedEvent.java index 2dfcc1db79..4bf03c90d9 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountRemovedEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountRemovedEvent.java @@ -19,16 +19,19 @@ package org.sleuthkit.autopsy.casemodule.events; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.datamodel.OsAccount; +import org.sleuthkit.autopsy.events.AutopsyEvent; /** * Event published when an OsAccount is deleted. + * + * oldValue will contain the objectId of the account that was removed. newValue + * will be null. */ -public final class OsAccountRemovedEvent extends OsAccountEvent { +public final class OsAccountRemovedEvent extends AutopsyEvent { private static final long serialVersionUID = 1L; - public OsAccountRemovedEvent(OsAccount account) { - super(Case.Events.OS_ACCOUNT_REMOVED.toString(), account); + public OsAccountRemovedEvent(Long osAccountObjectId) { + super(Case.Events.OS_ACCOUNT_REMOVED.toString(), osAccountObjectId, null); } } From e7cce9069e01170f6fc875dee6f69ca09ce5aff5 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 26 Mar 2021 12:49:04 -0400 Subject: [PATCH 07/17] 7415 file pipeline file save batching --- .../autopsy/ingest/FileIngestPipeline.java | 97 ++++++++++--------- 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 72716fc353..b27ffb23da 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -34,9 +34,13 @@ import org.sleuthkit.datamodel.TskCoreException; * A pipeline of file ingest modules for performing file ingest tasks for an * ingest job. */ +@NbBundle.Messages({ + "FileIngestPipeline_SaveResults_Activity=Saving Results" +}) final class FileIngestPipeline extends IngestTaskPipeline { private static final int FILE_BATCH_SIZE = 500; + private static final String SAVE_RESULTS_ACTIVITY = Bundle.FileIngestPipeline_SaveResults_Activity(); private static final IngestManager ingestManager = IngestManager.getInstance(); private final IngestJobPipeline ingestJobPipeline; private final List fileBatch; @@ -70,36 +74,15 @@ final class FileIngestPipeline extends IngestTaskPipeline { } @Override - @NbBundle.Messages({ - "FileIngestPipeline_SaveResults_Activity=Saving Results" - }) void completeTask(FileIngestTask task) throws IngestTaskPipelineException { - ingestManager.setIngestTaskProgress(task, Bundle.FileIngestPipeline_SaveResults_Activity()); - /* - * Close and cache the file from the file ingest task. The cache will be - * used for an eventual batch update of the case database with new - * properties added to the files in the cache by the ingest modules that - * processed them. - * - * Only one file ingest thread at a time will try to access the file - * cache. The synchronization here is to ensure visibility of the files - * in all of the threads that share the cache, rather than to prevent - * simultaneous access in multiple threads. - */ - synchronized (fileBatch) { - AbstractFile file = null; - try { - file = task.getFile(); - file.close(); - } catch (TskCoreException ex) { - throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS - } - if (!ingestJobPipeline.isCancelled()) { - fileBatch.add(file); - if (fileBatch.size() >= FILE_BATCH_SIZE) { - clearFileCache(); - } - } + try { + ingestManager.setIngestTaskProgress(task, SAVE_RESULTS_ACTIVITY); + AbstractFile file = task.getFile(); + file.close(); + cacheFileForBatchUpdate(file); + } catch (TskCoreException ex) { + throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS + } finally { ingestManager.setIngestTaskProgressCompleted(task); } } @@ -107,20 +90,44 @@ final class FileIngestPipeline extends IngestTaskPipeline { @Override List shutDown() { List errors = new ArrayList<>(); - if (!ingestJobPipeline.isCancelled()) { - Date start = new Date(); - try { - clearFileCache(); - } catch (IngestTaskPipelineException ex) { - errors.add(new IngestModuleError(Bundle.FileIngestPipeline_SaveResults_Activity(), ex)); - } - Date finish = new Date(); - ingestManager.incrementModuleRunTime(Bundle.FileIngestPipeline_SaveResults_Activity(), finish.getTime() - start.getTime()); + Date start = new Date(); + try { + clearFileCache(); + } catch (IngestTaskPipelineException ex) { + errors.add(new IngestModuleError(SAVE_RESULTS_ACTIVITY, ex)); } + Date finish = new Date(); + ingestManager.incrementModuleRunTime(SAVE_RESULTS_ACTIVITY, finish.getTime() - start.getTime()); errors.addAll(super.shutDown()); return errors; } + /** + * Adds a file to a file cache used to update the case database with new + * properties added to the files in the cache by the ingest modules that + * processed them. If adding the file to the cache fills the cache, a batch + * update is done immediately. + * + * @param file The file. + * + * @throws IngestTaskPipelineException Exception thrown if the case database + * update fails. + */ + private void cacheFileForBatchUpdate(AbstractFile file) throws IngestTaskPipelineException { + /* + * Only one file ingest thread at a time will try to access the file + * cache. The synchronization here is to ensure visibility of the files + * in all of the threads that share the cache, rather than to prevent + * simultaneous access in multiple threads. + */ + synchronized (fileBatch) { + fileBatch.add(file); + if (fileBatch.size() >= FILE_BATCH_SIZE) { + clearFileCache(); + } + } + } + /** * Updates the case database with new properties added to the files in the * cache by the ingest modules that processed them. @@ -138,16 +145,14 @@ final class FileIngestPipeline extends IngestTaskPipeline { synchronized (fileBatch) { CaseDbTransaction transaction = null; try { - Case currentCase = Case.getCurrentCaseThrows(); - SleuthkitCase caseDb = currentCase.getSleuthkitCase(); - transaction = caseDb.beginTransaction(); - for (AbstractFile file : fileBatch) { - if (!ingestJobPipeline.isCancelled()) { + if (!ingestJobPipeline.isCancelled()) { + Case currentCase = Case.getCurrentCaseThrows(); + SleuthkitCase caseDb = currentCase.getSleuthkitCase(); + transaction = caseDb.beginTransaction(); + for (AbstractFile file : fileBatch) { file.save(transaction); } - } - transaction.commit(); - if (!ingestJobPipeline.isCancelled()) { + transaction.commit(); for (AbstractFile file : fileBatch) { IngestManager.getInstance().fireFileIngestDone(file); } From 61e8bcaf4207de33811282138af2fe92b76276e7 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Fri, 26 Mar 2021 13:02:56 -0400 Subject: [PATCH 08/17] solr warning dialog will center now --- .../autopsy/casemodule/SolrNotConfiguredDialog.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/SolrNotConfiguredDialog.java b/Core/src/org/sleuthkit/autopsy/casemodule/SolrNotConfiguredDialog.java index 3adf647596..5fecaa9826 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/SolrNotConfiguredDialog.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/SolrNotConfiguredDialog.java @@ -37,11 +37,8 @@ class SolrNotConfiguredDialog extends javax.swing.JDialog { SolrNotConfiguredDialog() { super((JFrame) WindowManager.getDefault().getMainWindow(), true); // Center the startup window. - Dimension screenDimension = Toolkit.getDefaultToolkit().getScreenSize(); - int width = getSize().width; - int height = getSize().height; - setLocation((screenDimension.width - width) / 2, (screenDimension.height - height) / 2); initComponents(); + setLocationRelativeTo(WindowManager.getDefault().getMainWindow()); setIconImage(ImageUtilities.loadImage("org/sleuthkit/autopsy/images/warning16.png", false)); } From 1b07a81c752f093e726e8d3cdf55c212f1fc54ba Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 26 Mar 2021 13:37:34 -0400 Subject: [PATCH 09/17] 7415 file pipeline file save batching --- .../sleuthkit/autopsy/ingest/FileIngestPipeline.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index b27ffb23da..33718c2ad0 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -22,9 +22,11 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; @@ -41,6 +43,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { private static final int FILE_BATCH_SIZE = 500; private static final String SAVE_RESULTS_ACTIVITY = Bundle.FileIngestPipeline_SaveResults_Activity(); + private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName()); private static final IngestManager ingestManager = IngestManager.getInstance(); private final IngestJobPipeline ingestJobPipeline; private final List fileBatch; @@ -110,8 +113,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { * * @param file The file. * - * @throws IngestTaskPipelineException Exception thrown if the case database - * update fails. + * @throws IngestTaskPipelineException if the case database update fails. */ private void cacheFileForBatchUpdate(AbstractFile file) throws IngestTaskPipelineException { /* @@ -132,8 +134,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { * Updates the case database with new properties added to the files in the * cache by the ingest modules that processed them. * - * @throws IngestTaskPipelineException Exception thrown if the case database - * update fails. + * @throws IngestTaskPipelineException if the case database update fails. */ private void clearFileCache() throws IngestTaskPipelineException { /* @@ -161,7 +162,8 @@ final class FileIngestPipeline extends IngestTaskPipeline { if (transaction != null) { try { transaction.rollback(); - } catch (TskCoreException ignored) { + } catch (TskCoreException ex1) { + logger.log(Level.SEVERE, "Error rolling back transaction after failure to save updated properties for cached files from tasks", ex1); } } throw new IngestTaskPipelineException("Failed to save updated properties for cached files from tasks", ex); //NON-NLS From 29c1a51ddb64ef9fdf3d426ced0ed42a32b65a4d Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 26 Mar 2021 14:56:42 -0400 Subject: [PATCH 10/17] change to getAddr --- .../org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 1c0db38074..41a5a6dd89 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -1141,7 +1141,7 @@ class ExtractRegistry extends Extract { List existingAccounts = accountMgr.getAccounts(host); for(OsAccount osAccount: existingAccounts) { - Optional optional = osAccount.getUniqueIdWithinRealm(); + Optional optional = osAccount.getAddr(); if(!optional.isPresent()) { continue; } From f78e59e22d08f4fb998179c4be404af74f7e7c0a Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 29 Mar 2021 11:33:00 -0400 Subject: [PATCH 11/17] getId for Host,Person,Realm changed --- .../casemodule/AddImageWizardSelectHostVisual.java | 6 +++--- .../autopsy/casemodule/events/HostsEvent.java | 2 +- .../autopsy/casemodule/events/OsAccountEvent.java | 2 +- .../autopsy/casemodule/events/PersonsEvent.java | 2 +- .../contentviewers/osaccount/OsAccountDataPanel.java | 2 +- .../sleuthkit/autopsy/datamodel/HostDataSources.java | 6 +++--- .../sleuthkit/autopsy/datamodel/HostGrouping.java | 6 +++--- .../org/sleuthkit/autopsy/datamodel/HostNode.java | 4 ++-- .../sleuthkit/autopsy/datamodel/PersonGrouping.java | 6 +++--- .../autopsy/datamodel/PersonGroupingNode.java | 4 ++-- .../autopsy/datamodel/hosts/ManageHostsDialog.java | 12 ++++++------ .../autopsy/recentactivity/RAOsAccountCache.java | 2 +- 12 files changed, 27 insertions(+), 27 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardSelectHostVisual.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardSelectHostVisual.java index 9197b2fb42..4ab37d01b5 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardSelectHostVisual.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardSelectHostVisual.java @@ -79,7 +79,7 @@ class AddImageWizardSelectHostVisual extends javax.swing.JPanel { @Override public int hashCode() { int hash = 7; - hash = 41 * hash + Objects.hashCode(this.host == null ? 0 : this.host.getId()); + hash = 41 * hash + Objects.hashCode(this.host == null ? 0 : this.host.getHostId()); return hash; } @@ -96,8 +96,8 @@ class AddImageWizardSelectHostVisual extends javax.swing.JPanel { } final HostListItem other = (HostListItem) obj; if (!Objects.equals( - this.host == null ? 0 : this.host.getId(), - other.host == null ? 0 : other.host.getId())) { + this.host == null ? 0 : this.host.getHostId(), + other.host == null ? 0 : other.host.getHostId())) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java index 1ba225f4a2..f37a159125 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java @@ -42,7 +42,7 @@ public class HostsEvent extends TskDataModelChangeEvent { private static List getIds(List hosts) { return getSafeList(hosts).stream() .filter(h -> h != null) - .map(h -> h.getId()).collect(Collectors.toList()); + .map(h -> h.getHostId()).collect(Collectors.toList()); } /** diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java index 22b0622ba8..d34da7822d 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java @@ -56,7 +56,7 @@ class OsAccountEvent extends TskDataModelChangeEvent { @Override protected List getDataModelObjects(SleuthkitCase caseDb, List ids) throws TskCoreException { Long id = ids.get(0); - OsAccount account = caseDb.getOsAccountManager().getOsAccount(id); + OsAccount account = caseDb.getOsAccountManager().getOsAccountByObjectId(id); List accounts = new ArrayList<>(); accounts.add(account); return accounts; diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsEvent.java index ef74585b7f..e3f584d58a 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/PersonsEvent.java @@ -42,7 +42,7 @@ public class PersonsEvent extends TskDataModelChangeEvent { private static List getIds(List persons) { return getSafeList(persons).stream() .filter(h -> h != null) - .map(h -> h.getId()).collect(Collectors.toList()); + .map(h -> h.getPersonId()).collect(Collectors.toList()); } /** diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java index a1077e3e82..28d4ec3a24 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java @@ -362,7 +362,7 @@ public class OsAccountDataPanel extends JPanel { // Add attribute lists to the hostMap for (Host host : hosts) { - List atList = idMap.get(host.getId()); + List atList = idMap.get(host.getHostId()); if (atList != null) { hostMap.put(host, atList); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/HostDataSources.java b/Core/src/org/sleuthkit/autopsy/datamodel/HostDataSources.java index 37dadc840e..9451e401da 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/HostDataSources.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/HostDataSources.java @@ -51,7 +51,7 @@ public class HostDataSources implements AutopsyVisitableItem, Comparable h != null && h.getId() == hostId) + .filter(h -> h != null && h.getHostId() == hostId) .findFirst() .ifPresent((newHost) -> { setName(newHost.getName()); @@ -242,7 +242,7 @@ public class HostNode extends DisplayableItemNode { super(children, host == null ? Lookups.fixed(displayName) : Lookups.fixed(host, displayName)); - hostId = host == null ? null : host.getId(); + hostId = host == null ? null : host.getHostId(); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.HOSTS_CHANGED), WeakListeners.propertyChange(hostChangePcl, this)); super.setName(displayName); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGrouping.java b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGrouping.java index d606d288f1..1e65a06271 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/PersonGrouping.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/PersonGrouping.java @@ -52,7 +52,7 @@ public class PersonGrouping implements AutopsyVisitableItem, Comparable p != null && p.getId() == personId) + .filter(p -> p != null && p.getPersonId() == personId) .findFirst() .ifPresent((newPerson) -> { setName(newPerson.getName()); @@ -191,7 +191,7 @@ public class PersonGroupingNode extends DisplayableItemNode { super.setDisplayName(displayName); this.setIconBaseWithExtension(ICON_PATH); this.person = person; - this.personId = person == null ? null : person.getId(); + this.personId = person == null ? null : person.getPersonId(); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.PERSONS_CHANGED), WeakListeners.propertyChange(personChangePcl, this)); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/hosts/ManageHostsDialog.java b/Core/src/org/sleuthkit/autopsy/datamodel/hosts/ManageHostsDialog.java index 6d27cd53a1..7d4d521b51 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/hosts/ManageHostsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/hosts/ManageHostsDialog.java @@ -89,7 +89,7 @@ public class ManageHostsDialog extends javax.swing.JDialog { @Override public int hashCode() { int hash = 5; - hash = 89 * hash + Objects.hashCode(this.host == null ? 0 : this.host.getId()); + hash = 89 * hash + Objects.hashCode(this.host == null ? 0 : this.host.getHostId()); return hash; } @@ -109,7 +109,7 @@ public class ManageHostsDialog extends javax.swing.JDialog { return this.host == null && other.getHost() == null; } - return this.host.getId() == other.getHost().getId(); + return this.host.getHostId() == other.getHost().getHostId(); } } @@ -167,7 +167,7 @@ public class ManageHostsDialog extends javax.swing.JDialog { Long selectedId = null; try { Host newHost = Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().createHost(newHostName); - selectedId = newHost == null ? null : newHost.getId(); + selectedId = newHost == null ? null : newHost.getHostId(); } catch (NoCurrentCaseException | TskCoreException e) { logger.log(Level.WARNING, String.format("Unable to add new host '%s' at this time.", newHostName), e); } @@ -215,7 +215,7 @@ public class ManageHostsDialog extends javax.swing.JDialog { continue; } - if (host.getId() == selectedId) { + if (host.getHostId() == selectedId) { hostList.setSelectedIndex(i); return; } @@ -238,11 +238,11 @@ public class ManageHostsDialog extends javax.swing.JDialog { try { Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().updateHost(selectedHost); } catch (NoCurrentCaseException | TskCoreException e) { - logger.log(Level.WARNING, String.format("Unable to update host '%s' with id: %d at this time.", selectedHost.getName(), selectedHost.getId()), e); + logger.log(Level.WARNING, String.format("Unable to update host '%s' with id: %d at this time.", selectedHost.getName(), selectedHost.getHostId()), e); } HostListItem selectedItem = hostList.getSelectedValue(); - Long selectedId = selectedItem == null || selectedItem.getHost() == null ? null : selectedItem.getHost().getId(); + Long selectedId = selectedItem == null || selectedItem.getHost() == null ? null : selectedItem.getHost().getHostId(); refresh(); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java index f5208807af..8550f6d412 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java @@ -120,7 +120,7 @@ final class RAOsAccountCache { for (OsAccountAttribute attribute : attributeList) { if (attribute.getHostId().isPresent() - && attribute.getHostId().get().equals(host.getId()) + && attribute.getHostId().get().equals(host.getHostId()) && attribute.getAttributeType().equals(homeDir)) { accountCache.put(attribute.getValueString(), account); } From c3c9e2ead40c43bc66bc72278e4968c02759d6ed Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 29 Mar 2021 12:33:53 -0400 Subject: [PATCH 12/17] os account manager consistency --- .../org/sleuthkit/autopsy/datamodel/OsAccounts.java | 4 ++-- .../autopsy/recentactivity/ExtractRegistry.java | 12 ++++++------ .../autopsy/recentactivity/RAOsAccountCache.java | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java index 8465b76947..556b247164 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java @@ -144,10 +144,10 @@ public final class OsAccounts implements AutopsyVisitableItem { if(skCase != null) { try { if (filteringDSObjId == 0) { - list.addAll(skCase.getOsAccountManager().getAccounts()); + list.addAll(skCase.getOsAccountManager().getOsAccounts()); } else { Host host = skCase.getHostManager().getHost(skCase.getDataSource(filteringDSObjId)); - list.addAll(skCase.getOsAccountManager().getAccounts(host)); + list.addAll(skCase.getOsAccountManager().getOsAccounts(host)); } } catch (TskCoreException | TskDataException ex) { logger.log(Level.SEVERE, "Unable to retrieve list of OsAccounts for case", ex); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 41a5a6dd89..3af9e520a7 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -1139,7 +1139,7 @@ class ExtractRegistry extends Extract { HostManager hostMrg = tskCase.getHostManager(); Host host = hostMrg.getHost((DataSource)dataSource); - List existingAccounts = accountMgr.getAccounts(host); + List existingAccounts = accountMgr.getOsAccounts(host); for(OsAccount osAccount: existingAccounts) { Optional optional = osAccount.getAddr(); if(!optional.isPresent()) { @@ -1155,7 +1155,7 @@ class ExtractRegistry extends Extract { //add remaining userinfos as accounts; for (Map userInfo : userInfoMap.values()) { - OsAccount osAccount = accountMgr.createWindowsAccount(userInfo.get(SID_KEY), null, null, host, OsAccountRealm.RealmScope.UNKNOWN); + OsAccount osAccount = accountMgr.createWindowsOsAccount(userInfo.get(SID_KEY), null, null, host, OsAccountRealm.RealmScope.UNKNOWN); accountMgr.createOsAccountInstance(osAccount, (DataSource)dataSource, OsAccountInstance.OsAccountInstanceType.LAUNCHED); updateOsAccount(osAccount, userInfo, groupMap.get(userInfo.get(SID_KEY)), regAbstractFile); } @@ -2221,10 +2221,10 @@ class ExtractRegistry extends Extract { HostManager hostMrg = tskCase.getHostManager(); Host host = hostMrg.getHost((DataSource)dataSource); - Optional optional = accountMgr.getWindowsAccount(sid, null, null, host); + Optional optional = accountMgr.getWindowsOsAccount(sid, null, null, host); OsAccount osAccount; if (!optional.isPresent()) { - osAccount = accountMgr.createWindowsAccount(sid, userName != null && userName.isEmpty() ? null : userName, null, host, OsAccountRealm.RealmScope.UNKNOWN); + osAccount = accountMgr.createWindowsOsAccount(sid, userName != null && userName.isEmpty() ? null : userName, null, host, OsAccountRealm.RealmScope.UNKNOWN); accountMgr.createOsAccountInstance(osAccount, (DataSource)dataSource, OsAccountInstance.OsAccountInstanceType.LAUNCHED); } else { osAccount = optional.get(); @@ -2241,7 +2241,7 @@ class ExtractRegistry extends Extract { osAccount.addAttributes(attributes); } - accountMgr.updateAccount(osAccount); + accountMgr.updateOsAccount(osAccount); } /** @@ -2411,7 +2411,7 @@ class ExtractRegistry extends Extract { } osAccount.addAttributes(attributes); - tskCase.getOsAccountManager().updateAccount(osAccount); + tskCase.getOsAccountManager().updateOsAccount(osAccount); } /** diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java index f5208807af..8872421a93 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java @@ -113,7 +113,7 @@ final class RAOsAccountCache { */ private void buildAccountMap(SleuthkitCase tskCase, Host host) throws TskCoreException { BlackboardAttribute.Type homeDir = new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HOME_DIR); - List accounts = tskCase.getOsAccountManager().getAccounts(host); + List accounts = tskCase.getOsAccountManager().getOsAccounts(host); for (OsAccount account : accounts) { List attributeList = account.getOsAccountAttributes(); From 2a6908486942bec8dfa00fbbf2d2fe3f6ebfda64 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 29 Mar 2021 13:04:25 -0400 Subject: [PATCH 13/17] getOsAccountbyObjectid changes --- .../org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java index d34da7822d..e08b4a406e 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java @@ -56,7 +56,7 @@ class OsAccountEvent extends TskDataModelChangeEvent { @Override protected List getDataModelObjects(SleuthkitCase caseDb, List ids) throws TskCoreException { Long id = ids.get(0); - OsAccount account = caseDb.getOsAccountManager().getOsAccountByObjectId(id); + OsAccount account = caseDb.getOsAccountManager().getOsAccountbyObjectid(id); List accounts = new ArrayList<>(); accounts.add(account); return accounts; From 8061e8c0bf7342ef30343e92696c5febcafe2347 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 29 Mar 2021 13:24:17 -0400 Subject: [PATCH 14/17] 7415 Change FileIngestPipeline method name --- .../org/sleuthkit/autopsy/ingest/FileIngestPipeline.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 33718c2ad0..f7f8a8f555 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -95,7 +95,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { List errors = new ArrayList<>(); Date start = new Date(); try { - clearFileCache(); + updateBatchedFiles(); } catch (IngestTaskPipelineException ex) { errors.add(new IngestModuleError(SAVE_RESULTS_ACTIVITY, ex)); } @@ -125,7 +125,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { synchronized (fileBatch) { fileBatch.add(file); if (fileBatch.size() >= FILE_BATCH_SIZE) { - clearFileCache(); + updateBatchedFiles(); } } } @@ -136,7 +136,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { * * @throws IngestTaskPipelineException if the case database update fails. */ - private void clearFileCache() throws IngestTaskPipelineException { + private void updateBatchedFiles() throws IngestTaskPipelineException { /* * Only one file ingest thread at a time will try to access the file * cache. The synchronization here is to ensure visibility of the files From 66a8d1634b1b4f65f9d31d20ad0497e21daa3a84 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 29 Mar 2021 15:14:02 -0400 Subject: [PATCH 15/17] getOsAccountByObjectId changes --- .../org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java index e08b4a406e..d34da7822d 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/OsAccountEvent.java @@ -56,7 +56,7 @@ class OsAccountEvent extends TskDataModelChangeEvent { @Override protected List getDataModelObjects(SleuthkitCase caseDb, List ids) throws TskCoreException { Long id = ids.get(0); - OsAccount account = caseDb.getOsAccountManager().getOsAccountbyObjectid(id); + OsAccount account = caseDb.getOsAccountManager().getOsAccountByObjectId(id); List accounts = new ArrayList<>(); accounts.add(account); return accounts; From 11ddee1c7232eb8a6127d2c014f8a21ad7719cdc Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 29 Mar 2021 17:30:09 -0400 Subject: [PATCH 16/17] Change getAccounts calls to getOsAccounts so Autopsy builds --- .../org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java | 2 +- .../org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java index 72b8e0d38c..8e6ad2df8e 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java @@ -340,7 +340,7 @@ final class ExtractRecycleBin extends Extract { private Map makeUserNameMap(Content dataSource) throws TskCoreException { Map userNameMap = new HashMap<>(); - for(OsAccount account: tskCase.getOsAccountManager().getAccounts(((DataSource)dataSource).getHost())) { + for(OsAccount account: tskCase.getOsAccountManager().getOsAccounts(((DataSource)dataSource).getHost())) { Optional userName = account.getLoginName(); userNameMap.put(account.getName(), userName.isPresent() ? userName.get() : ""); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 348dc72ae4..1699042b52 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -1716,7 +1716,7 @@ class ExtractRegistry extends Extract { private Map makeUserNameMap(Content dataSource) throws TskCoreException { Map map = new HashMap<>(); - for(OsAccount account: tskCase.getOsAccountManager().getAccounts(((DataSource)dataSource).getHost())) { + for(OsAccount account: tskCase.getOsAccountManager().getOsAccounts(((DataSource)dataSource).getHost())) { Optional userName = account.getLoginName(); map.put(account.getName(), userName.isPresent() ? userName.get() : ""); } From f77a646a83f89e246a47cd78c83295dad0823d2e Mon Sep 17 00:00:00 2001 From: apriestman Date: Tue, 30 Mar 2021 09:40:03 -0400 Subject: [PATCH 17/17] Update name of getOsAccountByObjectId --- .../autopsy/contentviewers/osaccount/OsAccountDataPanel.java | 2 +- .../sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java | 2 +- .../src/org/sleuthkit/autopsy/recentactivity/Extract.java | 2 +- .../org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java index dcdb1e20d1..79b22d81cb 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java @@ -362,7 +362,7 @@ public class OsAccountDataPanel extends JPanel { OsAccountManager osAccountManager = Case.getCurrentCase().getSleuthkitCase().getOsAccountManager(); if(account == null) { - account = osAccountManager.getOsAccount(accountId); + account = osAccountManager.getOsAccountByObjectId(accountId); } List hosts = osAccountManager.getHosts(account); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java index ce60d94d7a..1ddc1ed401 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java @@ -541,7 +541,7 @@ final class ChromeCacheExtractor { Optional optional = cacheEntryFile.getOsAccountObjectId(); OsAccount account = null; if(optional.isPresent()) { - account = currentCase.getSleuthkitCase().getOsAccountManager().getOsAccount(optional.get()); + account = currentCase.getSleuthkitCase().getOsAccountManager().getOsAccountByObjectId(optional.get()); } BlackboardArtifact webCacheArtifact = cacheEntryFile.newDataArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_CACHE), webAttr, account); artifactsAdded.add(webCacheArtifact); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index d7b4bb20a2..e152a146fa 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -535,7 +535,7 @@ abstract class Extract { if(osAccountCache == null) { Optional accountId = ((AbstractFile)content).getOsAccountObjectId(); if(accountId.isPresent()) { - return Optional.ofNullable(tskCase.getOsAccountManager().getOsAccount(accountId.get())); + return Optional.ofNullable(tskCase.getOsAccountManager().getOsAccountByObjectId(accountId.get())); } return Optional.empty(); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java index f58c4488ea..788f219371 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java @@ -78,7 +78,7 @@ final class RAOsAccountCache { return getAccountForPath(file.getParentPath()); } - OsAccount osAccount = Case.getCurrentCase().getSleuthkitCase().getOsAccountManager().getOsAccount(optional.get()); + OsAccount osAccount = Case.getCurrentCase().getSleuthkitCase().getOsAccountManager().getOsAccountByObjectId(optional.get()); if (osAccount.getName().equals("S-1-5-32-544")) { return getAccountForPath(file.getParentPath()); }