diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardSelectHostVisual.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardSelectHostVisual.java index 4ab37d01b5..07eaf84104 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardSelectHostVisual.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardSelectHostVisual.java @@ -166,7 +166,7 @@ class AddImageWizardSelectHostVisual extends javax.swing.JPanel { if (specifyNewHostRadio.isSelected() && StringUtils.isNotEmpty(specifyNewHostTextField.getText())) { String newHostName = specifyNewHostTextField.getText(); try { - return Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().createHost(newHostName); + return Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().newHost(newHostName); } catch (NoCurrentCaseException | TskCoreException ex) { logger.log(Level.WARNING, String.format("Unable to create host '%s'.", newHostName), ex); return null; @@ -186,7 +186,7 @@ class AddImageWizardSelectHostVisual extends javax.swing.JPanel { */ private void loadHostData() { try { - Collection hosts = Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().getHosts(); + Collection hosts = Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().getAllHosts(); sanitizedHostSet = HostNameValidator.getSanitizedHostNames(hosts); Vector hostListItems = hosts.stream() diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java index f37a159125..efa60c8b47 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/events/HostsEvent.java @@ -76,7 +76,7 @@ public class HostsEvent extends TskDataModelChangeEvent { continue; } - Optional thisHostOpt = hostManager.getHost(id); + Optional thisHostOpt = hostManager.getHostById(id); thisHostOpt.ifPresent((h) -> toRet.add(h)); } } diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java index c1966cb067..d324c336b5 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java @@ -45,7 +45,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.OsAccount; -import org.sleuthkit.datamodel.OsAccountAttribute; +import org.sleuthkit.datamodel.OsAccount.OsAccountAttribute; import org.sleuthkit.datamodel.OsAccountInstance; import org.sleuthkit.datamodel.OsAccountManager; import org.sleuthkit.datamodel.OsAccountRealm; @@ -390,10 +390,10 @@ public class OsAccountDataPanel extends JPanel { account = osAccountManager.getOsAccountByObjectId(accountId); } - OsAccountRealm realm = skCase.getOsAccountRealmManager().getRealmById(account.getRealmId()); + OsAccountRealm realm = skCase.getOsAccountRealmManager().getRealmByRealmId(account.getRealmId()); List hosts = osAccountManager.getHosts(account); - List attributeList = account.getOsAccountAttributes(); + List attributeList = account.getExtendedOsAccountAttributes(); if (attributeList != null) { if (hosts != null) { diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.form b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.form index e627b40c94..1b6d64b8a1 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.form +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.form @@ -14,12 +14,17 @@ - + + + + + + - - + + diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java index dd43d1f531..2195e4fbdb 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java @@ -18,7 +18,10 @@ */ package org.sleuthkit.autopsy.contentviewers.osaccount; +import java.awt.BorderLayout; import java.awt.Component; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; import java.util.Optional; import java.util.logging.Level; import org.openide.nodes.Node; @@ -153,8 +156,14 @@ public class OsAccountViewer extends javax.swing.JPanel implements DataContentVi mainScrollPane = new javax.swing.JScrollPane(); - setLayout(new java.awt.BorderLayout()); - add(mainScrollPane, java.awt.BorderLayout.CENTER); + setLayout(new java.awt.GridBagLayout()); + + mainScrollPane.setPreferredSize(new java.awt.Dimension(200, 0)); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; + gridBagConstraints.weightx = 1.0; + gridBagConstraints.weighty = 1.0; + add(mainScrollPane, gridBagConstraints); }// //GEN-END:initComponents diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Installer.java b/Core/src/org/sleuthkit/autopsy/corecomponents/Installer.java index 5bcd68a03b..93378c6e19 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Installer.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Installer.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.logging.Level; import javax.swing.BorderFactory; +import javax.swing.ImageIcon; import javax.swing.SwingUtilities; import javax.swing.UIManager; import javax.swing.UIManager.LookAndFeelInfo; @@ -75,6 +76,16 @@ public class Installer extends ModuleInstall { } private void setLookAndFeel() { + + ImageIcon questionIcon = new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/question_32.png")); + ImageIcon warningIcon = new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/warning_32.png")); + ImageIcon informationIcon = new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/information_32.png")); + ImageIcon errorIcon = new ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/error_32.png")); + UIManager.put("OptionPane.errorIcon", errorIcon); + UIManager.put("OptionPane.warningIcon", warningIcon); + UIManager.put("OptionPane.questionIcon", questionIcon); + UIManager.put("OptionPane.informationIcon", informationIcon); + if (System.getProperty("os.name").toLowerCase().contains("mac")) { //NON-NLS setUnixLookAndFeel(); setModuleSettings("false"); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java index c909ddaad3..22b340d5f1 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AutopsyTreeChildFactory.java @@ -115,7 +115,7 @@ public final class AutopsyTreeChildFactory extends ChildFactory.Detachable toPopulate) { try { - Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().getHosts().stream() + Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().getAllHosts().stream() .map(HostDataSources::new) .sorted() .forEach(toPopulate::add); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java b/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java index 674568dd3e..58becb183c 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java @@ -32,6 +32,7 @@ import java.util.Observable; import java.util.Observer; import java.util.Set; import java.util.logging.Level; +import org.apache.commons.lang3.tuple.Pair; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; import org.openide.nodes.Node; @@ -77,7 +78,7 @@ public class EmailExtracted implements AutopsyVisitableItem { */ public static final Map parsePath(String path) { Map parsed = new HashMap<>(); - String[] split = path.split(MAIL_PATH_SEPARATOR); + String[] split = path == null ? new String[0] : path.split(MAIL_PATH_SEPARATOR); if (split.length < 4) { parsed.put(MAIL_ACCOUNT, NbBundle.getMessage(EmailExtracted.class, "EmailExtracted.defaultAcct.text")); parsed.put(MAIL_FOLDER, NbBundle.getMessage(EmailExtracted.class, "EmailExtracted.defaultFolder.text")); @@ -146,55 +147,59 @@ public class EmailExtracted implements AutopsyVisitableItem { } @SuppressWarnings("deprecation") - public void update() { - synchronized (accounts) { - accounts.clear(); - } + public void update() { + // clear cache if no case if (skCase == null) { + synchronized (accounts) { + accounts.clear(); + } return; } - int artId = BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(); + // get artifact id and path (if present) of all email artifacts + int emailArtifactId = BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(); int pathAttrId = BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH.getTypeID(); - String query = "SELECT value_text,blackboard_attributes.artifact_id,attribute_type_id " //NON-NLS - + "FROM blackboard_attributes,blackboard_artifacts WHERE " //NON-NLS - + "attribute_type_id=" + pathAttrId //NON-NLS - + " AND blackboard_attributes.artifact_id=blackboard_artifacts.artifact_id" //NON-NLS - + " AND blackboard_artifacts.artifact_type_id=" + artId; //NON-NLS - if (filteringDSObjId > 0) { - query += " AND blackboard_artifacts.data_source_obj_id = " + filteringDSObjId; - } - + + String query = "SELECT \n" + + " art.artifact_id AS artifact_id,\n" + + " (SELECT value_text FROM blackboard_attributes attr\n" + + " WHERE attr.artifact_id = art.artifact_id AND attr.attribute_type_id = " + pathAttrId + "\n" + + " LIMIT 1) AS value_text\n" + + "FROM \n" + + " blackboard_artifacts art\n" + + " WHERE art.artifact_type_id = " + emailArtifactId + "\n" + + ((filteringDSObjId > 0) ? " AND art.data_source_obj_id = " + filteringDSObjId : ""); + + // form hierarchy of account -> folder -> account id + Map>> newMapping = new HashMap<>(); + try (CaseDbQuery dbQuery = skCase.executeQuery(query)) { ResultSet resultSet = dbQuery.getResultSet(); - synchronized (accounts) { - while (resultSet.next()) { - final String path = resultSet.getString("value_text"); //NON-NLS - final long artifactId = resultSet.getLong("artifact_id"); //NON-NLS - final Map parsedPath = parsePath(path); - final String account = parsedPath.get(MAIL_ACCOUNT); - final String folder = parsedPath.get(MAIL_FOLDER); - - Map> folders = accounts.get(account); - if (folders == null) { - folders = new LinkedHashMap<>(); - accounts.put(account, folders); - } - List messages = folders.get(folder); - if (messages == null) { - messages = new ArrayList<>(); - folders.put(folder, messages); - } - messages.add(artifactId); - } + while (resultSet.next()) { + Long artifactId = resultSet.getLong("artifact_id"); + Map accountFolderMap = parsePath(resultSet.getString("value_text")); + String account = accountFolderMap.get(MAIL_ACCOUNT); + String folder = accountFolderMap.get(MAIL_FOLDER); + + Map> folders = newMapping.computeIfAbsent(account, (str) -> new LinkedHashMap<>()); + List messages = folders.computeIfAbsent(folder, (str) -> new ArrayList<>()); + messages.add(artifactId); } } catch (TskCoreException | SQLException ex) { logger.log(Level.WARNING, "Cannot initialize email extraction: ", ex); //NON-NLS } + + + synchronized (accounts) { + accounts.clear(); + accounts.putAll(newMapping); + } + setChanged(); notifyObservers(); } } + /** * Mail root node grouping all mail accounts, supports account-> folder diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java index 556b247164..4b2588c6bb 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java @@ -146,7 +146,7 @@ public final class OsAccounts implements AutopsyVisitableItem { if (filteringDSObjId == 0) { list.addAll(skCase.getOsAccountManager().getOsAccounts()); } else { - Host host = skCase.getHostManager().getHost(skCase.getDataSource(filteringDSObjId)); + Host host = skCase.getHostManager().getHostByDataSource(skCase.getDataSource(filteringDSObjId)); list.addAll(skCase.getOsAccountManager().getOsAccounts(host)); } } catch (TskCoreException | TskDataException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/hosts/AssociateNewPersonAction.java b/Core/src/org/sleuthkit/autopsy/datamodel/hosts/AssociateNewPersonAction.java index bea7f7b372..2cb89995c6 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/hosts/AssociateNewPersonAction.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/hosts/AssociateNewPersonAction.java @@ -65,7 +65,7 @@ public class AssociateNewPersonAction extends AbstractAction { try { newPersonName = getAddDialogName(); if (StringUtils.isNotBlank(newPersonName)) { - Person person = Case.getCurrentCaseThrows().getSleuthkitCase().getPersonManager().createPerson(newPersonName); + Person person = Case.getCurrentCaseThrows().getSleuthkitCase().getPersonManager().newPerson(newPersonName); Case.getCurrentCaseThrows().getSleuthkitCase().getPersonManager().setPerson(host, person); } } catch (NoCurrentCaseException | TskCoreException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/hosts/ManageHostsDialog.java b/Core/src/org/sleuthkit/autopsy/datamodel/hosts/ManageHostsDialog.java index 7d4d521b51..6cb4220a7a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/hosts/ManageHostsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/hosts/ManageHostsDialog.java @@ -166,7 +166,7 @@ public class ManageHostsDialog extends javax.swing.JDialog { if (newHostName != null) { Long selectedId = null; try { - Host newHost = Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().createHost(newHostName); + Host newHost = Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().newHost(newHostName); selectedId = newHost == null ? null : newHost.getHostId(); } catch (NoCurrentCaseException | TskCoreException e) { logger.log(Level.WARNING, String.format("Unable to add new host '%s' at this time.", newHostName), e); @@ -234,9 +234,8 @@ public class ManageHostsDialog extends javax.swing.JDialog { if (selectedHost != null) { String newHostName = getAddEditDialogName(selectedHost); if (newHostName != null) { - selectedHost.setName(newHostName); try { - Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().updateHost(selectedHost); + Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().updateHostName(selectedHost, newHostName); } catch (NoCurrentCaseException | TskCoreException e) { logger.log(Level.WARNING, String.format("Unable to update host '%s' with id: %d at this time.", selectedHost.getName(), selectedHost.getHostId()), e); } @@ -322,7 +321,7 @@ public class ManageHostsDialog extends javax.swing.JDialog { Map> hostMapping = new HashMap<>(); try { SleuthkitCase curCase = Case.getCurrentCaseThrows().getSleuthkitCase(); - List hosts = curCase.getHostManager().getHosts(); + List hosts = curCase.getHostManager().getAllHosts(); List dataSources = curCase.getDataSources(); if (dataSources != null) { diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/hosts/MergeHostMenuAction.java b/Core/src/org/sleuthkit/autopsy/datamodel/hosts/MergeHostMenuAction.java index 8b8ddc3f23..5517687434 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/hosts/MergeHostMenuAction.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/hosts/MergeHostMenuAction.java @@ -67,7 +67,7 @@ public class MergeHostMenuAction extends AbstractAction implements Presenter.Pop // Get a list of all other hosts List otherHosts = Collections.emptyList(); try { - otherHosts = Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().getHosts(); + otherHosts = Case.getCurrentCaseThrows().getSleuthkitCase().getHostManager().getAllHosts(); otherHosts.remove(sourceHost); } catch (NoCurrentCaseException | TskCoreException ex) { logger.log(Level.WARNING, "Error getting hosts for case.", ex); diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainFilterPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainFilterPanel.java index f2a5f688a9..683b2e6c95 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainFilterPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainFilterPanel.java @@ -18,9 +18,6 @@ */ package org.sleuthkit.autopsy.discovery.ui; -import java.util.ArrayList; -import java.util.List; -import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository; import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.discovery.search.DiscoveryAttributes; import org.sleuthkit.autopsy.discovery.search.ResultsSorter; @@ -46,14 +43,7 @@ public class DomainFilterPanel extends AbstractFiltersPanel { addFilter(new KnownAccountTypeFilterPanel(), false, null, 1); addFilter(new ArtifactTypeFilterPanel(), false, null, 1); addFilter(new DateFilterPanel(), false, null, 1); - List defaultFrequencies = null; - if (CentralRepository.isEnabled()) { - defaultFrequencies = new ArrayList<>(); - defaultFrequencies.add(SearchData.Frequency.RARE); - defaultFrequencies.add(SearchData.Frequency.UNIQUE); - defaultFrequencies.add(SearchData.Frequency.COMMON); - } - addFilter(new PastOccurrencesFilterPanel(TYPE), true, defaultFrequencies, 0); + addFilter(new PastOccurrencesFilterPanel(TYPE), false, null, 0); addPanelsToScrollPane(domainFiltersSplitPane); setLastGroupingAttributeType(DiscoveryAttributes.GroupingAttributeType.LAST_ACTIVITY_DATE); setLastSortingMethod(ResultsSorter.SortingMethod.BY_DOMAIN_NAME); diff --git a/Core/src/org/sleuthkit/autopsy/images/error_32.png b/Core/src/org/sleuthkit/autopsy/images/error_32.png new file mode 100755 index 0000000000..50c92f43bf Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/images/error_32.png differ diff --git a/Core/src/org/sleuthkit/autopsy/images/information_32.png b/Core/src/org/sleuthkit/autopsy/images/information_32.png new file mode 100755 index 0000000000..c00c1e7dc2 Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/images/information_32.png differ diff --git a/Core/src/org/sleuthkit/autopsy/images/question_32.png b/Core/src/org/sleuthkit/autopsy/images/question_32.png new file mode 100755 index 0000000000..8e0094a8ac Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/images/question_32.png differ diff --git a/Core/src/org/sleuthkit/autopsy/images/warning_32.png b/Core/src/org/sleuthkit/autopsy/images/warning_32.png new file mode 100755 index 0000000000..251c2eb88e Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/images/warning_32.png differ diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index ca2da24cdf..6a361fbbd1 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -77,6 +77,11 @@ import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper; import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CallMediaType; import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CommunicationDirection; import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.MessageReadStatus; +import org.sleuthkit.datamodel.blackboardutils.GeoArtifactsHelper; +import org.sleuthkit.datamodel.blackboardutils.attributes.GeoTrackPoints; +import org.sleuthkit.datamodel.blackboardutils.attributes.GeoTrackPoints.TrackPoint; +import org.sleuthkit.datamodel.blackboardutils.attributes.GeoWaypoints; +import org.sleuthkit.datamodel.blackboardutils.attributes.GeoWaypoints.Waypoint; import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments; import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.FileAttachment; import org.w3c.dom.Document; @@ -149,23 +154,41 @@ public final class LeappFileProcessor { .build(); private static final Map ACCOUNT_RELATIONSHIPS = ImmutableMap.builder() - .put("Zapya.tsv", "message") + .put("zapya.tsv", "message") .put("sms messages.tsv", "message") .put("mms messages.tsv", "message") - .put("Viber - Messages.tsv", "message") - .put("Viber - Contacts.tsv", "contact") - .put("Viber - Call Logs.tsv", "calllog") - .put("Xender file transfer - Messages.tsv", "message") - .put("Whatsapp - Contacts.tsv", "contact") - .put("Whatsapp - Group Call Logs.tsv", "calllog") - .put("Whatsapp - Single Call Logs.tsv", "calllog") - .put("Whatsapp - Messages Logs.tsv", "message") - .put("Shareit file transfer.tsv", "message") + .put("viber - messages.tsv", "message") + .put("viber - contacts.tsv", "contact") + .put("viber - call logs.tsv", "calllog") + .put("xender file transfer - messages.tsv", "message") + .put("xender file transfer - contacts.tsv", "contact") + .put("whatsapp - contacts.tsv", "contact") + .put("whatsapp - group call logs.tsv", "calllog") + .put("whatsapp - single call logs.tsv", "calllog") + .put("whatsapp - messages logs.tsv", "message") + .put("shareit file transfer.tsv", "message") .put("tangomessages messages.tsv", "message") + .put("contacts.tsv", "contact") + .put("imo - accountid.tsv", "contact") + .put("imo - messages.tsv", "message") + .put("textnow - contacts.tsv", "contact") + .put("textnow - messages.tsv", "message") + .put("line - messages.tsv", "message") + .put("line - contacts.tsv", "contact") + .put("line - calllogs.tsv", "calllog") + .put("skype - messages logs.tsv", "message") + .put("skype - contacts.tsv", "contact") + .put("skype - call logs.tsv", "calllog") + .put("facebook messenger - chats.tsv", "message") + .put("facebook messenger - contacts.tsv", "contact") + .put("facebook messenger - calls.tsv", "calllog") + .put("call logs2.tsv", "calllog") + .put("call logs.tsv", "calllog") + .put("oruxmaps tracks.tsv", "trackpoint") + .put("google map locations.tsv", "route") .put("Contacts.tsv", "contact") - .put("IMO - AccountId.tsv", "contact") - .put("IMO - messages.tsv", "message") - + .put("sms - imessage.tsv", "message") + .put("call history.tsv", "calllog") .build(); Blackboard blkBoard; @@ -318,6 +341,10 @@ public final class LeappFileProcessor { List bbartifacts, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException, TskCoreException { + String trackpointSegmentName = null; + GeoTrackPoints pointList = new GeoTrackPoints(); + AbstractFile geoAbstractFile = null; + if (LeappFile == null || !LeappFile.exists() || fileName == null) { logger.log(Level.WARNING, String.format("Leap file: %s is null or does not exist", LeappFile == null ? LeappFile.toString() : "")); return; @@ -348,7 +375,7 @@ public final class LeappFileProcessor { Collection bbattributes = processReadLine(columnItems, columnIndexes, attrList, fileName, lineNum); if (!bbattributes.isEmpty()) { - switch (ACCOUNT_RELATIONSHIPS.getOrDefault(fileName, "norelationship").toLowerCase()) { + switch (ACCOUNT_RELATIONSHIPS.getOrDefault(fileName.toLowerCase(), "norelationship").toLowerCase()) { case "message": createMessageRelationship(bbattributes, dataSource, fileName); break; @@ -358,6 +385,12 @@ public final class LeappFileProcessor { case "calllog": createCalllogRelationship(bbattributes, dataSource, fileName); break; + case "route": + createRoute(bbattributes, dataSource, fileName); + break; + case "trackpoint": + geoAbstractFile = createTrackpoint(bbattributes, dataSource, fileName, trackpointSegmentName, pointList); + break; default: // There is no relationship defined so just process the artifact normally BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), dataSource, bbattributes); if (bbartifact != null) { @@ -371,8 +404,158 @@ public final class LeappFileProcessor { } } } + + try { + if (ACCOUNT_RELATIONSHIPS.getOrDefault(fileName.toLowerCase(), "norelationship").toLowerCase() == "trackpoint") { + (new GeoArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, "", geoAbstractFile)).addTrack(trackpointSegmentName, pointList, new ArrayList<>()); + + } + } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) { + throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS + } + } + @NbBundle.Messages({ + "LeappFileProcessor.cannot.create.waypoint.relationship=Cannot create TSK_WAYPOINT artifact.", + }) + + private void createRoute (Collection bbattributes, Content dataSource, String fileName) throws IngestModuleException { + + Double startLatitude = Double.valueOf(0); + Double startLongitude = Double.valueOf(0); + Double endLatitude = Double.valueOf(0); + Double endLongitude = Double.valueOf(0); + Double zeroValue = Double.valueOf(0); + String destinationName = ""; + String locationName = ""; + Long dateTime = Long.valueOf(0); + Collection otherAttributes = new ArrayList<>(); + String sourceFile = null; + AbstractFile absFile = null; + String comment = ""; + + try { + for (BlackboardAttribute bba : bbattributes) { + switch (bba.getAttributeType().getTypeName()) { + case "TSK_GEO_LATITUDE_START": + startLatitude = bba.getValueDouble(); + break; + case "TSK_GEO_LONGITUDE_START": + startLongitude = bba.getValueDouble(); + break; + case "TSK_GEO_LATITUDE_END": + startLatitude = bba.getValueDouble(); + break; + case "TSK_GEO_LONGITUDE_END": + startLongitude = bba.getValueDouble(); + break; + case "TSK_DATETIME": + dateTime = bba.getValueLong(); + break; + case "TSK_NAME": + destinationName = bba.getValueString(); + break; + case "TSK_LOCATION": + locationName = bba.getValueString(); + break; + case "TSK_TEXT_FILE": + sourceFile = bba.getValueString(); + break; + case "TSK_COMMENT": + comment = bba.getValueString(); + break; + default: + otherAttributes.add(bba); + break; + } + } + absFile = findAbstractFile(dataSource, sourceFile); + if (absFile == null) { + absFile = (AbstractFile) dataSource; + } + GeoWaypoints waypointList = new GeoWaypoints(); + waypointList.addPoint(new Waypoint(startLatitude, startLongitude, zeroValue, "")); + waypointList.addPoint(new Waypoint(endLatitude, endLongitude, zeroValue, locationName)); + (new GeoArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, comment, absFile)).addRoute(destinationName, dateTime, waypointList, new ArrayList<>()); + + } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) { + throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_waypoint_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS + } + + + } + + @NbBundle.Messages({ + "LeappFileProcessor.cannot.create.trackpoint.relationship=Cannot create TSK_TRACK_POINT artifact.", + }) + + private AbstractFile createTrackpoint(Collection bbattributes, Content dataSource, String fileName, String trackpointSegmentName, GeoTrackPoints pointList) throws IngestModuleException { + + Double latitude = Double.valueOf(0); + Double longitude = Double.valueOf(0); + Double altitude = Double.valueOf(0); + Double zeroValue = Double.valueOf(0); + String segmentName = null; + Long dateTime = Long.valueOf(0); + Collection otherAttributes = new ArrayList<>(); + String sourceFile = null; + String comment = null; + AbstractFile absFile = null; + + try { + for (BlackboardAttribute bba : bbattributes) { + switch (bba.getAttributeType().getTypeName()) { + case "TSK_GEO_LATITUDE": + latitude = bba.getValueDouble(); + break; + case "TSK_GEO_LONGITUDE": + longitude = bba.getValueDouble(); + break; + case "TSK_GEO_ALTITUDE": + altitude = bba.getValueDouble(); + break; + case "TSK_DATETIME": + dateTime = bba.getValueLong(); + break; + case "TSK_NAME": + segmentName = bba.getValueString(); + break; + case "TSK_TEXT_FILE": + sourceFile = bba.getValueString(); + break; + case "TSK_COMMENT": + comment = bba.getValueString(); + otherAttributes.add(bba); + break; + default: + otherAttributes.add(bba); + break; + } + } + absFile = findAbstractFile(dataSource, sourceFile); + if (absFile == null) { + absFile = (AbstractFile) dataSource; + } + if ((trackpointSegmentName == null) || (trackpointSegmentName == segmentName)) { + trackpointSegmentName = segmentName; + pointList.addPoint(new TrackPoint(latitude, longitude, altitude, segmentName, zeroValue, zeroValue, zeroValue, dateTime)); + } else { + (new GeoArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, comment, absFile)).addTrack(segmentName, pointList, new ArrayList<>()); + trackpointSegmentName = segmentName; + pointList = new GeoTrackPoints(); + pointList.addPoint(new TrackPoint(latitude, longitude, altitude, segmentName, zeroValue, zeroValue, zeroValue, dateTime)); + + } + } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) { + throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_trackpoint_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS + } + + return absFile; + + } + + @NbBundle.Messages({ "LeappFileProcessor.cannot.create.message.relationship=Cannot create TSK_MESSAGE Relationship.", }) @@ -380,6 +563,7 @@ public final class LeappFileProcessor { private void createMessageRelationship(Collection bbattributes, Content dataSource, String fileName) throws IngestModuleException { String messageType = null; + String alternateId = null; CommunicationDirection communicationDirection = CommunicationDirection.UNKNOWN; String senderId = null; String receipentId = null; @@ -441,23 +625,34 @@ public final class LeappFileProcessor { case "TSK_SUBJECT": subject = bba.getValueString(); break; + case "TSK_ID": + alternateId = bba.getValueString(); + otherAttributes.add(bba); + break; default: otherAttributes.add(bba); break; } } AbstractFile absFile = findAbstractFile(dataSource, sourceFile); - Account.Type accountType = getAccountType(fileName); - if ((absFile != null) || (accountType != null)) { - CommunicationArtifactsHelper accountArtifact = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), + if (absFile == null) { + absFile = (AbstractFile) dataSource; + } + CommunicationArtifactsHelper accountArtifact; + Account.Type accountType = getAccountType(fileName); + if (alternateId == null) { + accountArtifact = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), moduleName, absFile, accountType); - BlackboardArtifact messageArtifact = accountArtifact.addMessage(messageType, communicationDirection, senderId, - receipentId, dateTime, messageStatus, subject, - messageText, threadId, otherAttributes); - if (!fileAttachments.isEmpty()) { - messageAttachments = new MessageAttachments(fileAttachments, new ArrayList<>()); - accountArtifact.addAttachments(messageArtifact, messageAttachments); - } + } else { + accountArtifact = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), + moduleName, absFile, accountType, accountType, alternateId); + } + BlackboardArtifact messageArtifact = accountArtifact.addMessage(messageType, communicationDirection, senderId, + receipentId, dateTime, messageStatus, subject, + messageText, threadId, otherAttributes); + if (!fileAttachments.isEmpty()) { + messageAttachments = new MessageAttachments(fileAttachments, new ArrayList<>()); + accountArtifact.addAttachments(messageArtifact, messageAttachments); } } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) { throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS @@ -465,6 +660,9 @@ public final class LeappFileProcessor { } + @NbBundle.Messages({ + "LeappFileProcessor.cannot.create.contact.relationship=Cannot create TSK_CONTACT Relationship.", + }) private void createContactRelationship(Collection bbattributes, Content dataSource, String fileName) throws IngestModuleException { String alternateId = null; @@ -503,6 +701,7 @@ public final class LeappFileProcessor { break; case "TSK_ID": alternateId = bba.getValueString(); + otherAttributes.add(bba); break; default: otherAttributes.add(bba); @@ -510,8 +709,11 @@ public final class LeappFileProcessor { } } AbstractFile absFile = findAbstractFile(dataSource, sourceFile); + if (absFile == null) { + absFile = (AbstractFile) dataSource; + } Account.Type accountType = getAccountType(fileName); - if ((absFile != null) || (accountType != null)) { + if (accountType != null) { CommunicationArtifactsHelper accountArtifact; if (alternateId == null) { @@ -524,14 +726,18 @@ public final class LeappFileProcessor { BlackboardArtifact messageArtifact = accountArtifact.addContact(contactName, phoneNumber, homePhoneNumber, mobilePhoneNumber, emailAddr, otherAttributes); } } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) { - throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS + throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_contact_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS } - } + @NbBundle.Messages({ + "LeappFileProcessor.cannot.create.calllog.relationship=Cannot create TSK_CALLLOG Relationship.", + }) + private void createCalllogRelationship(Collection bbattributes, Content dataSource, String fileName) throws IngestModuleException { String callerId = null; + String alternateId = null; List calleeId = Arrays.asList(); CommunicationDirection communicationDirection = CommunicationDirection.UNKNOWN; Long startDateTime = Long.valueOf(0); @@ -570,6 +776,10 @@ public final class LeappFileProcessor { calleeId = Arrays.asList(calleeTempList); } break; + case "TSK_ID": + alternateId = bba.getValueString(); + otherAttributes.add(bba); + break; default: otherAttributes.add(bba); break; @@ -582,14 +792,21 @@ public final class LeappFileProcessor { callerId = null; } AbstractFile absFile = findAbstractFile(dataSource, sourceFile); - Account.Type accountType = getAccountType(fileName); - if ((absFile != null) || (accountType != null)) { - CommunicationArtifactsHelper accountArtifact = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), - moduleName, absFile, accountType); - BlackboardArtifact callLogArtifact = accountArtifact.addCalllog(communicationDirection, callerId, calleeId, startDateTime, endDateTime, mediaType, otherAttributes); + if (absFile == null) { + absFile = (AbstractFile) dataSource; } + Account.Type accountType = getAccountType(fileName); + CommunicationArtifactsHelper accountArtifact; + if (accountType != null) { + accountArtifact = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), + moduleName, absFile, accountType); + } else { + accountArtifact = new CommunicationArtifactsHelper(Case.getCurrentCaseThrows().getSleuthkitCase(), + moduleName, absFile, accountType, accountType, alternateId); + } + BlackboardArtifact callLogArtifact = accountArtifact.addCalllog(communicationDirection, callerId, calleeId, startDateTime, endDateTime, mediaType, otherAttributes); } catch (NoCurrentCaseException | TskCoreException | BlackboardException ex) { - throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS + throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_calllog_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS } } @@ -606,6 +823,10 @@ public final class LeappFileProcessor { return Account.Type.IMO; case "imo - messages.tsv": return Account.Type.IMO; + case "textnow - contacts.tsv": + return Account.Type.TEXTNOW; + case "textnow - messages.tsv": + return Account.Type.TEXTNOW; case "mms messages.tsv": return Account.Type.PHONE; case "viber - call logs.tsv": @@ -616,6 +837,8 @@ public final class LeappFileProcessor { return Account.Type.VIBER; case "xender file transfer - messages.tsv": return Account.Type.XENDER; + case "xender file transfer - contacts.tsv": + return Account.Type.XENDER; case "whatsapp - single call logs.tsv": return Account.Type.WHATSAPP; case "whatsapp - messages logs.tsv": @@ -627,9 +850,33 @@ public final class LeappFileProcessor { case "tangomessages messages.tsv": return Account.Type.TANGO; case "shareit file transfer.tsv": - return Account.Type.SHAREIT; + return Account.Type.SHAREIT; + case "line - calllogs.tsv": + return Account.Type.LINE; + case "line - contacts.tsv": + return Account.Type.LINE; + case "line - messages.tsv": + return Account.Type.LINE; + case "skype - call logs.tsv": + return Account.Type.SKYPE; + case "skype - contacts.tsv": + return Account.Type.SKYPE; + case "skype - messages logs.tsv": + return Account.Type.SKYPE; + case "facebook messenger - calls.tsv": + return Account.Type.FACEBOOK; + case "facebook messenger - contacts.tsv": + return Account.Type.FACEBOOK; + case "facebook messenger - chats.tsv": + return Account.Type.FACEBOOK; + case "call logs2.tsv": + return Account.Type.PHONE; + case "call logs.tsv": + return Account.Type.PHONE; + case "sms - imessage.tsv": + return Account.Type.PHONE; default: - return null; + return Account.Type.PHONE; } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleap-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleap-artifact-attribute-reference.xml index 27d3bc7263..3ef61b0025 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleap-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleap-artifact-attribute-reference.xml @@ -355,15 +355,6 @@ - - @@ -379,8 +370,9 @@ - - + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileap-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileap-artifact-attribute-reference.xml index 16409f42c7..70725fd0c6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileap-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileap-artifact-attribute-reference.xml @@ -50,9 +50,9 @@ - + - + @@ -101,6 +101,7 @@ + @@ -743,16 +744,14 @@ - diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java index 56bb8e9133..f8dbebe224 100644 --- a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java @@ -1085,7 +1085,7 @@ public class PortableCaseReportModule implements ReportModule { Host newHost = null; if (content instanceof DataSource) { Host oldHost = ((DataSource)content).getHost(); - newHost = portableSkCase.getHostManager().createHost(oldHost.getName()); + newHost = portableSkCase.getHostManager().newHost(oldHost.getName()); } CaseDbTransaction trans = portableSkCase.beginTransaction(); diff --git a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java index 07d1d39308..f2e7654908 100644 --- a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java @@ -70,12 +70,13 @@ import org.sleuthkit.datamodel.ReadContentInputStream; import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap; import java.io.InputStreamReader; import java.nio.charset.Charset; import java.util.ArrayList; import org.apache.tika.parser.pdf.PDFParserConfig.OCR_STRATEGY; import org.sleuthkit.autopsy.coreutils.ExecUtil.HybridTerminator; +import org.sleuthkit.datamodel.TskData; /** * Extracts text from Tika supported content. Protects against Tika parser hangs @@ -133,7 +134,7 @@ final class TikaTextExtractor implements TextExtractor { // Used to log to the tika file that is why it uses the java.util.logging.logger class instead of the Autopsy one private static final java.util.logging.Logger TIKA_LOGGER = java.util.logging.Logger.getLogger("Tika"); //NON-NLS private static final Logger AUTOPSY_LOGGER = Logger.getLogger(TikaTextExtractor.class.getName()); - + private static final int LIMITED_OCR_SIZE_MIN = 100 * 1024; private final ThreadFactory tikaThreadFactory = new ThreadFactoryBuilder().setNameFormat("tika-reader-%d").build(); private final ExecutorService executorService = Executors.newSingleThreadExecutor(tikaThreadFactory); @@ -143,6 +144,7 @@ final class TikaTextExtractor implements TextExtractor { private final Content content; private boolean tesseractOCREnabled; + private boolean limitedOCREnabled; private static final String TESSERACT_DIR_NAME = "Tesseract-OCR"; //NON-NLS private static final String TESSERACT_EXECUTABLE = "tesseract.exe"; //NON-NLS private static final File TESSERACT_PATH = locateTesseractExecutable(); @@ -158,7 +160,7 @@ final class TikaTextExtractor implements TextExtractor { .map(mt -> mt.getType() + "/" + mt.getSubtype()) .collect(Collectors.toList()); - public TikaTextExtractor(Content content) { + TikaTextExtractor(Content content) { this.content = content; } @@ -198,7 +200,7 @@ final class TikaTextExtractor implements TextExtractor { // Handle images seperately so the OCR task can be cancelled. // See JIRA-4519 for the need to have cancellation in the UI and ingest. - if (ocrEnabled() && mimeType.toLowerCase().startsWith("image/")) { + if (ocrEnabled() && mimeType.toLowerCase().startsWith("image/") && useOcrOnFile(file)) { InputStream imageOcrStream = performOCR(file); return new InputStreamReader(imageOcrStream, Charset.forName("UTF-8")); } @@ -219,7 +221,7 @@ final class TikaTextExtractor implements TextExtractor { officeParserConfig.setUseSAXDocxExtractor(true); parseContext.set(OfficeParserConfig.class, officeParserConfig); - if (ocrEnabled()) { + if (ocrEnabled() && useOcrOnFile(file)) { // Configure OCR for Tika if it chooses to run OCR // during extraction TesseractOCRConfig ocrConfig = new TesseractOCRConfig(); @@ -256,7 +258,7 @@ final class TikaTextExtractor implements TextExtractor { + "Tika returned empty reader for " + content); } pushbackReader.unread(read); - + //Save the metadata if it has not been fetched already. if (metadataMap == null) { metadataMap = new HashMap<>(); @@ -264,7 +266,7 @@ final class TikaTextExtractor implements TextExtractor { metadataMap.put(mtdtKey, metadata.get(mtdtKey)); } } - + return new ReaderCharSource(pushbackReader).openStream(); } catch (TimeoutException te) { final String msg = NbBundle.getMessage(this.getClass(), @@ -345,6 +347,22 @@ final class TikaTextExtractor implements TextExtractor { } } + /** + * Method to indicate if OCR should be performed on this image file. Checks + * to see if the limited OCR setting is enabled. If it is it will also check + * that one of the limiting factors is true. + * + * @param file The AbstractFile which OCR might be performed on. + * @param boolean The configuration setting which indicates if limited OCR + * is enabled in Keyword Search. + * + * @return True if limited OCR is not enabled or the image is greater than + * 100KB in size or the image is a derived file. + */ + private boolean useOcrOnFile(AbstractFile file) { + return !limitedOCREnabled || file.getSize() > LIMITED_OCR_SIZE_MIN || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED; + } + /** * Wraps the creation of a TikaReader into a Future so that it can be * cancelled. @@ -356,7 +374,7 @@ final class TikaTextExtractor implements TextExtractor { private final Metadata metadata; private final ParseContext parseContext; - public GetTikaReader(AutoDetectParser parser, InputStream stream, + GetTikaReader(AutoDetectParser parser, InputStream stream, Metadata metadata, ParseContext parseContext) { this.parser = parser; this.stream = stream; @@ -386,7 +404,7 @@ final class TikaTextExtractor implements TextExtractor { * * @throws FileNotFoundException */ - public CleanUpStream(File file) throws FileNotFoundException { + CleanUpStream(File file) throws FileNotFoundException { super(file); this.file = file; } @@ -442,7 +460,7 @@ final class TikaTextExtractor implements TextExtractor { if (metadataMap != null) { return ImmutableMap.copyOf(metadataMap); } - + try { metadataMap = new HashMap<>(); InputStream stream = new ReadContentInputStream(content); @@ -528,7 +546,7 @@ final class TikaTextExtractor implements TextExtractor { * @param context Instance containing config classes */ @Override - public void setExtractionSettings(Lookup context) { + public void setExtractionSettings(Lookup context) { if (context != null) { List terminators = new ArrayList<>(); ImageConfig configInstance = context.lookup(ImageConfig.class); @@ -536,11 +554,13 @@ final class TikaTextExtractor implements TextExtractor { if (Objects.nonNull(configInstance.getOCREnabled())) { this.tesseractOCREnabled = configInstance.getOCREnabled(); } - + if (Objects.nonNull(configInstance.getLimitedOCREnabled())) { + this.limitedOCREnabled = configInstance.getLimitedOCREnabled(); + } if (Objects.nonNull(configInstance.getOCRLanguages())) { this.languagePacks = formatLanguagePacks(configInstance.getOCRLanguages()); } - + terminators.add(configInstance.getOCRTimeoutTerminator()); } @@ -548,8 +568,8 @@ final class TikaTextExtractor implements TextExtractor { if (terminatorInstance != null) { terminators.add(terminatorInstance); } - - if(!terminators.isEmpty()) { + + if (!terminators.isEmpty()) { this.processTerminator = new HybridTerminator(terminators); } } @@ -572,4 +592,4 @@ final class TikaTextExtractor implements TextExtractor { return reader; } } -} \ No newline at end of file +} diff --git a/Core/src/org/sleuthkit/autopsy/textextractors/configs/ImageConfig.java b/Core/src/org/sleuthkit/autopsy/textextractors/configs/ImageConfig.java index a92852101c..6c501a7bdf 100755 --- a/Core/src/org/sleuthkit/autopsy/textextractors/configs/ImageConfig.java +++ b/Core/src/org/sleuthkit/autopsy/textextractors/configs/ImageConfig.java @@ -29,10 +29,11 @@ import org.sleuthkit.autopsy.coreutils.ExecUtil.TimedProcessTerminator; * @see org.openide.util.Lookup */ public class ImageConfig { - + private static final int OCR_TIMEOUT_SECONDS = 30 * 60; private Boolean OCREnabled; + private Boolean limitedOCREnabled; private List ocrLanguages; private final TimedProcessTerminator ocrTimedTerminator = new TimedProcessTerminator(OCR_TIMEOUT_SECONDS); @@ -46,6 +47,16 @@ public class ImageConfig { this.OCREnabled = enabled; } + /** + * Enables the limiting OCR to be run on larger images and images which were + * extracted from documents. + * + * @param enabled Flag indicating if OCR is enabled. + */ + public void setLimitedOCREnabled(boolean enabled) { + this.limitedOCREnabled = enabled; + } + /** * Gets the OCR flag that has been set. By default this flag is turned off. * @@ -57,9 +68,9 @@ public class ImageConfig { /** * Sets languages for OCR. - * + * * See PlatformUtil for list of installed language packs. - * + * * @param languages List of languages to use */ public void setOCRLanguages(List languages) { @@ -68,19 +79,30 @@ public class ImageConfig { /** * Gets the list of languages OCR should perform. - * + * * @return Collection of OCR languages */ public List getOCRLanguages() { return this.ocrLanguages; } - + /** * Returns a ProcessTerminator for timing out the OCR process. - * + * * @return ProcessTerminator instance. */ public ProcessTerminator getOCRTimeoutTerminator() { return ocrTimedTerminator; } + + /** + * Gets the limited OCR flag to indicate if OCR should be limited to larger + * images and images which were extracted from documents. + * + * @return Flag indicating if limited OCR is enabled. True if OCR should be + * limited, false otherwise.. + */ + public boolean getLimitedOCREnabled() { + return limitedOCREnabled; + } } diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/translators/BingTranslator.java b/Core/src/org/sleuthkit/autopsy/texttranslation/translators/BingTranslator.java index 948aae9217..6524c1835d 100644 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/translators/BingTranslator.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/translators/BingTranslator.java @@ -30,6 +30,7 @@ import java.io.IOException; import javax.swing.JPanel; import org.openide.util.NbBundle.Messages; import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.texttranslation.TextTranslator; import org.sleuthkit.autopsy.texttranslation.TranslationConfigException; import org.sleuthkit.autopsy.texttranslation.TranslationException; @@ -46,7 +47,7 @@ public class BingTranslator implements TextTranslator { //https://docs.microsoft.com/en-us/azure/cognitive-services/translator/language-support private static final String BASE_URL = "https://api.cognitive.microsofttranslator.com/translate?api-version=3.0&to="; private static final int MAX_STRING_LENGTH = 5000; - private final BingTranslatorSettingsPanel settingsPanel; + private BingTranslatorSettingsPanel settingsPanel; private final BingTranslatorSettings settings = new BingTranslatorSettings(); // This sends messages to Microsoft. private final OkHttpClient CLIENT = new OkHttpClient(); @@ -55,11 +56,11 @@ public class BingTranslator implements TextTranslator { * Create a Bing Translator */ public BingTranslator() { - settingsPanel = new BingTranslatorSettingsPanel(settings.getAuthenticationKey(), settings.getTargetLanguageCode()); + } /** - * Get the tranlationurl for the specified language code + * Get the tranlation url for the specified language code * * * @@ -133,7 +134,11 @@ public class BingTranslator implements TextTranslator { } @Override + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) public JPanel getSettingsPanel() { + if(settingsPanel == null) { + settingsPanel = new BingTranslatorSettingsPanel(settings.getAuthenticationKey(), settings.getTargetLanguageCode()); + } return settingsPanel; } diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/translators/GoogleTranslator.java b/Core/src/org/sleuthkit/autopsy/texttranslation/translators/GoogleTranslator.java index ec74190229..c0820b688c 100644 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/translators/GoogleTranslator.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/translators/GoogleTranslator.java @@ -36,6 +36,7 @@ import org.apache.commons.lang3.StringUtils; import org.openide.util.NbBundle.Messages; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.coreutils.EscapeUtil; +import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.texttranslation.TextTranslator; import org.sleuthkit.autopsy.texttranslation.TranslationConfigException; import org.sleuthkit.autopsy.texttranslation.TranslationException; @@ -50,7 +51,7 @@ public final class GoogleTranslator implements TextTranslator { private static final Logger logger = Logger.getLogger(GoogleTranslator.class.getName()); //See translate method for justification of this limit. private static final int MAX_PAYLOAD_SIZE = 5000; - private final GoogleTranslatorSettingsPanel settingsPanel; + private GoogleTranslatorSettingsPanel settingsPanel; private final GoogleTranslatorSettings settings = new GoogleTranslatorSettings(); private Translate googleTranslate; @@ -59,7 +60,6 @@ public final class GoogleTranslator implements TextTranslator { */ public GoogleTranslator() { // Instantiates a client - settingsPanel = new GoogleTranslatorSettingsPanel(settings.getCredentialPath(), settings.getTargetLanguageCode()); loadTranslator(); } @@ -134,7 +134,11 @@ public final class GoogleTranslator implements TextTranslator { } @Override + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) public JPanel getSettingsPanel() { + if(settingsPanel == null) { + settingsPanel = new GoogleTranslatorSettingsPanel(settings.getCredentialPath(), settings.getTargetLanguageCode()); + } return settingsPanel; } diff --git a/Core/src/org/sleuthkit/autopsy/threadutils/TaskRetryUtil.java b/Core/src/org/sleuthkit/autopsy/threadutils/TaskRetryUtil.java index e00cedab83..79484dddb1 100755 --- a/Core/src/org/sleuthkit/autopsy/threadutils/TaskRetryUtil.java +++ b/Core/src/org/sleuthkit/autopsy/threadutils/TaskRetryUtil.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2020 Basis Technology Corp. + * Copyright 2020-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -163,7 +163,7 @@ public class TaskRetryUtil { * each attempt and an optional timeout for each attempt. If an attempt * times out, that particular attempt task will be cancelled. * - * @tparam T The return type of the task. + * @tparam T The return type of the task. * @param task The task. * @param attempts The defining details for each attempt of the task. * @param executor The scheduled task executor to be used to attempt the @@ -181,6 +181,9 @@ public class TaskRetryUtil { * @throws InterruptedException */ public static T attemptTask(Callable task, List attempts, ScheduledThreadPoolExecutor executor, Terminator terminator, Logger logger, String taskDesc) throws InterruptedException { + /* + * Attempt the task. + */ T result = null; String taskDescForLog = taskDesc != null ? taskDesc : "Task"; int attemptCounter = 0; @@ -195,9 +198,6 @@ public class TaskRetryUtil { break; } TaskAttempt attempt = attempts.get(attemptCounter); - if (logger != null) { - logger.log(Level.INFO, String.format("SCHEDULING '%s' (attempt = %d, delay = %d %s, timeout = %d %s)", taskDescForLog, attemptCounter + 1, attempt.getDelay(), attempt.getTimeUnit(), attempt.getTimeout(), attempt.getTimeUnit())); - } if (attemptCounter > 0) { totalTaskRetries.incrementAndGet(); } @@ -222,11 +222,27 @@ public class TaskRetryUtil { } ++attemptCounter; } + + /* + * If the task required more than one attempt, log it. + */ + if (logger != null && attemptCounter > 1) { + if (result != null) { + logger.log(Level.WARNING, String.format("'%s' succeeded after %d attempts", taskDescForLog, attemptCounter)); + } else { + logger.log(Level.SEVERE, String.format("'%s' failed after %d attempts", taskDescForLog, attemptCounter)); + } + } + + /* + * If the task failed, count it as a failed task. + */ if (result == null) { if (terminator == null || !terminator.stopTaskAttempts()) { totalFailedTasks.incrementAndGet(); } } + return result; } diff --git a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java index 067a5fceea..5b2932b718 100644 --- a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java +++ b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java @@ -45,7 +45,6 @@ import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.openide.modules.InstalledFileLocator; -import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.url.analytics.DomainCategory; /** @@ -396,7 +395,7 @@ class WebCategoriesDataModel implements AutoCloseable { * @return The list of domain suffixes and their categories. * @throws SQLException */ - List getRecords() throws SQLException { + synchronized List getRecords() throws SQLException { if (!isInitialized()) { initialize(); } @@ -428,7 +427,7 @@ class WebCategoriesDataModel implements AutoCloseable { * @return The found entry or null. * @throws SQLException */ - DomainCategory getRecordBySuffix(String domainSuffix) throws SQLException { + synchronized DomainCategory getRecordBySuffix(String domainSuffix) throws SQLException { if (!isInitialized()) { initialize(); } @@ -529,7 +528,9 @@ class WebCategoriesDataModel implements AutoCloseable { @Override public synchronized void close() throws SQLException { - dbConn.close(); - dbConn = null; + if (dbConn != null) { + dbConn.close(); + dbConn = null; + } } } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index b3a25be677..a15fb3b50b 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -973,7 +973,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen * job to be shut down in an orderly fashion. */ void cancelCurrentJob() { - if (State.RUNNING != state) { + if ((State.RUNNING != state) && (State.SHUTTING_DOWN != state)) { return; } synchronized (jobsLock) { @@ -2564,6 +2564,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen synchronized (ingestLock) { // Try each DSP in decreasing order of confidence for (AutoIngestDataSourceProcessor selectedProcessor : validDataSourceProcessors) { + currentJob.setDataSourceProcessor(selectedProcessor); UUID taskId = UUID.randomUUID(); caseForJob.notifyAddingDataSource(taskId); DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId, ingestLock); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED index 823399e0d0..56a675e256 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED @@ -205,9 +205,7 @@ DeleteCaseTask.progress.parsingManifest=Parsing manifest file {0}... DeleteCaseTask.progress.releasingManifestLock=Releasing lock on the manifest file {0}... DeleteCaseTask.progress.startMessage=Starting deletion... DeleteOrphanCaseNodesAction.progressDisplayName=Cleanup Case Znodes -# {0} - item count DeleteOrphanCaseNodesDialog.additionalInit.lblNodeCount.text=Znodes found: {0} -# {0} - item count DeleteOrphanCaseNodesDialog.additionalInit.znodesTextArea.countMessage=ZNODES FOUND: {0} DeleteOrphanCaseNodesTask.progress.connectingToCoordSvc=Connecting to the coordination service # {0} - node path diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index d2ab5f76c7..7dfee92048 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -320,3 +320,4 @@ ExtractedContentPanel.pageOfLabel.text=of ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) +KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text=Only process images which are over 100KB in size or extracted from a document (Beta) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 72de1c17d8..c60c34bade 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -23,6 +23,7 @@ ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try aga ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress ExtractAllTermsReport.startExport=Starting Unique Word Extraction ExtractedContentPanel.setMarkup.panelTxt=Loading text... Please wait +# {0} - Content name ExtractedContentPanel.SetMarkup.progress.loading=Loading text for {0} GlobalEditListPanel.editKeyword.title=Edit Keyword GlobalEditListPanel.warning.text=Boundary characters ^ and $ do not match word boundaries. Consider\nreplacing with an explicit list of boundary characters, such as [ \\.,] @@ -30,6 +31,7 @@ GlobalEditListPanel.warning.title=Warning IndexedText.errorMessage.errorGettingText=Error retrieving indexed text. IndexedText.warningMessage.knownFile=This file is a known file (based on MD5 hash) and does not have indexed text. IndexedText.warningMessage.noTextAvailable=No indexed text for this file. +KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsLimitedOCR=Only process images which are over 100KB in size or extracted from a document. (Beta) (Requires Windows 64-bit) KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsOCR=Enable Optical Character Recognition (OCR) (Requires Windows 64-bit) KeywordSearchGlobalSettingsPanel.Title=Global Keyword Search Settings KeywordSearchIngestModule.init.badInitMsg=Keyword search server was not properly initialized, cannot run keyword search ingest. @@ -49,7 +51,7 @@ KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search @@ -400,6 +402,7 @@ ExtractedContentPanel.pageOfLabel.text=of ExtractedContentPanel.pageCurLabel.text=- ExtractedContentPanel.pagesLabel.text=Page: KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR) +KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text=Only process images which are over 100KB in size or extracted from a document (Beta) TextZoomPanel.zoomInButton.text= TextZoomPanel.zoomOutButton.text= TextZoomPanel.zoomResetButton.text=Reset diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form index 4625056adf..550ad5d442 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form @@ -31,22 +31,17 @@ - - - - - - - - - - + + + + + @@ -54,7 +49,7 @@ - + @@ -68,6 +63,10 @@ + + + + @@ -92,6 +91,8 @@ + + @@ -121,7 +122,7 @@ - + @@ -288,5 +289,15 @@ + + + + + + + + + + diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java index 0741d8d6bb..b9687a32c2 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java @@ -35,7 +35,8 @@ import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.UpdateFrequ */ @SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implements OptionsPanel { - + + private static final long serialVersionUID = 1L; private final Logger logger = Logger.getLogger(KeywordSearchGlobalSearchSettingsPanel.class.getName()); /** @@ -45,17 +46,19 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen initComponents(); customizeComponents(); } - + private void activateWidgets() { skipNSRLCheckBox.setSelected(KeywordSearchSettings.getSkipKnown()); showSnippetsCB.setSelected(KeywordSearchSettings.getShowSnippets()); ocrCheckBox.setSelected(KeywordSearchSettings.getOcrOption()); + limitedOcrCheckbox.setSelected(KeywordSearchSettings.getLimitedOcrOption()); boolean ingestRunning = IngestManager.getInstance().isIngestRunning(); ingestWarningLabel.setVisible(ingestRunning); skipNSRLCheckBox.setEnabled(!ingestRunning); ocrCheckBox.setEnabled(!ingestRunning); + limitedOcrCheckbox.setEnabled(ocrCheckBox.isSelected() && !ingestRunning); setTimeSettingEnabled(!ingestRunning); - + final UpdateFrequency curFreq = KeywordSearchSettings.getUpdateFrequency(); switch (curFreq) { case FAST: @@ -109,6 +112,7 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen timeRadioButton5 = new javax.swing.JRadioButton(); ingestWarningLabel = new javax.swing.JLabel(); ocrCheckBox = new javax.swing.JCheckBox(); + limitedOcrCheckbox = new javax.swing.JCheckBox(); skipNSRLCheckBox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text")); // NOI18N skipNSRLCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.toolTipText")); // NOI18N @@ -189,6 +193,13 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen } }); + limitedOcrCheckbox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text")); // NOI18N + limitedOcrCheckbox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + limitedOcrCheckboxActionPerformed(evt); + } + }); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -203,26 +214,23 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen .addComponent(settingsLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(settingsSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 326, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addGroup(layout.createSequentialGroup() - .addGap(10, 10, 10) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(skipNSRLCheckBox) - .addComponent(showSnippetsCB) - .addComponent(ocrCheckBox))) .addGroup(layout.createSequentialGroup() .addComponent(informationLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(informationSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 309, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGroup(layout.createSequentialGroup() - .addGap(10, 10, 10) + .addGap(16, 16, 16) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(skipNSRLCheckBox) + .addComponent(showSnippetsCB) + .addComponent(ocrCheckBox) .addGroup(layout.createSequentialGroup() .addComponent(filesIndexedLabel) .addGap(18, 18, 18) .addComponent(filesIndexedValue)) .addComponent(frequencyLabel) .addGroup(layout.createSequentialGroup() - .addGap(10, 10, 10) + .addGap(16, 16, 16) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(timeRadioButton2) .addComponent(timeRadioButton1) @@ -232,7 +240,10 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen .addGroup(layout.createSequentialGroup() .addComponent(chunksLabel) .addGap(18, 18, 18) - .addComponent(chunksValLabel))))) + .addComponent(chunksValLabel)) + .addGroup(layout.createSequentialGroup() + .addGap(16, 16, 16) + .addComponent(limitedOcrCheckbox))))) .addGap(0, 0, Short.MAX_VALUE))) .addContainerGap()) ); @@ -252,6 +263,8 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen .addComponent(showSnippetsCB) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(ocrCheckBox) + .addGap(0, 0, 0) + .addComponent(limitedOcrCheckbox) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(frequencyLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) @@ -278,7 +291,7 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen .addComponent(chunksValLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(ingestWarningLabel) - .addContainerGap(43, Short.MAX_VALUE)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); }// //GEN-END:initComponents @@ -311,9 +324,14 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen }//GEN-LAST:event_timeRadioButton4ActionPerformed private void ocrCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_ocrCheckBoxActionPerformed + limitedOcrCheckbox.setEnabled(ocrCheckBox.isSelected()); firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); }//GEN-LAST:event_ocrCheckBoxActionPerformed + private void limitedOcrCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_limitedOcrCheckboxActionPerformed + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + }//GEN-LAST:event_limitedOcrCheckboxActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JLabel chunksLabel; private javax.swing.JLabel chunksValLabel; @@ -323,6 +341,7 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen private javax.swing.JLabel informationLabel; private javax.swing.JSeparator informationSeparator; private javax.swing.JLabel ingestWarningLabel; + private javax.swing.JCheckBox limitedOcrCheckbox; private javax.swing.JCheckBox ocrCheckBox; private javax.swing.JLabel settingsLabel; private javax.swing.JSeparator settingsSeparator; @@ -342,13 +361,14 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen KeywordSearchSettings.setUpdateFrequency(getSelectedTimeValue()); KeywordSearchSettings.setShowSnippets(showSnippetsCB.isSelected()); KeywordSearchSettings.setOcrOption(ocrCheckBox.isSelected()); + KeywordSearchSettings.setLimitedOcrOption(limitedOcrCheckbox.isSelected()); } - + @Override public void load() { activateWidgets(); } - + private void setTimeSettingEnabled(boolean enabled) { timeRadioButton1.setEnabled(enabled); timeRadioButton2.setEnabled(enabled); @@ -357,7 +377,7 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen timeRadioButton5.setEnabled(enabled); frequencyLabel.setEnabled(enabled); } - + private UpdateFrequency getSelectedTimeValue() { if (timeRadioButton1.isSelected()) { return UpdateFrequency.FAST; @@ -372,18 +392,19 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen } return UpdateFrequency.DEFAULT; } - - @NbBundle.Messages({"KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsOCR=Enable Optical Character Recognition (OCR) (Requires Windows 64-bit)"}) + + @NbBundle.Messages({"KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsOCR=Enable Optical Character Recognition (OCR) (Requires Windows 64-bit)", + "KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsLimitedOCR=Only process images which are over 100KB in size or extracted from a document. (Beta) (Requires Windows 64-bit)"}) private void customizeComponents() { - + timeGroup.add(timeRadioButton1); timeGroup.add(timeRadioButton2); timeGroup.add(timeRadioButton3); timeGroup.add(timeRadioButton4); timeGroup.add(timeRadioButton5); - + this.skipNSRLCheckBox.setSelected(KeywordSearchSettings.getSkipKnown()); - + try { filesIndexedValue.setText(Integer.toString(KeywordSearch.getServer().queryNumIndexedFiles())); chunksValLabel.setText(Integer.toString(KeywordSearch.getServer().queryNumIndexedChunks())); @@ -395,15 +416,18 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen ocrCheckBox.setText(Bundle.KeywordSearchGlobalSearchSettingsPanel_customizeComponents_windowsOCR()); ocrCheckBox.setSelected(false); ocrCheckBox.setEnabled(false); - } - + limitedOcrCheckbox.setSelected(false); + limitedOcrCheckbox.setEnabled(false); + limitedOcrCheckbox.setText(Bundle.KeywordSearchGlobalSearchSettingsPanel_customizeComponents_windowsLimitedOCR()); + } + KeywordSearch.addNumIndexedFilesChangeListener( new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); Object newValue = evt.getNewValue(); - + if (changed.equals(KeywordSearch.NUM_FILES_CHANGE_EVT)) { int newFilesIndexed = ((Integer) newValue); filesIndexedValue.setText(Integer.toString(newFilesIndexed)); @@ -416,7 +440,7 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen } } }); - + //allow panel to toggle its enabled status while it is open based on ingest events IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { @Override diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 4ef6db2a24..0d8e48e9c4 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -513,6 +513,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { private boolean extractTextAndIndex(AbstractFile aFile, Map extractedMetadata) throws IngesterException { ImageConfig imageConfig = new ImageConfig(); imageConfig.setOCREnabled(KeywordSearchSettings.getOcrOption()); + imageConfig.setLimitedOCREnabled(KeywordSearchSettings.getLimitedOcrOption()); ProcessTerminator terminator = () -> context.fileIngestIsCancelled(); Lookup extractionContext = Lookups.fixed(imageConfig, terminator); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java index 91043ee9a0..0db0b30684 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java @@ -39,9 +39,11 @@ class KeywordSearchSettings { static final String PROPERTIES_NSRL = NbBundle.getMessage(KeywordSearchSettings.class, "KeywordSearchSettings.propertiesNSRL.text", MODULE_NAME); static final String PROPERTIES_SCRIPTS = NbBundle.getMessage(KeywordSearchSettings.class, "KeywordSearchSettings.propertiesScripts.text", MODULE_NAME); static final String SHOW_SNIPPETS = "showSnippets"; //NON-NLS - static final boolean DEFAULT_SHOW_SNIPPETS = true; + static final boolean DEFAULT_SHOW_SNIPPETS = true; static final String OCR_ENABLED = "ocrEnabled"; //NON-NLS + static final String LIMITED_OCR_ENABLED = "limitedOcrEnabled"; //NON-NLS static final boolean OCR_ENABLED_DEFAULT = false; // NON-NLS + static final boolean LIMITED_OCR_ENABLED_DEFAULT = false; private static boolean skipKnown = true; private static final Logger logger = Logger.getLogger(KeywordSearchSettings.class.getName()); private static UpdateFrequency UpdateFreq = UpdateFrequency.DEFAULT; @@ -130,19 +132,19 @@ class KeywordSearchSettings { stringExtractOptions.put(key, val); ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, key, val); } - + /** * Save OCR setting to permanent storage - * + * * @param enabled Is OCR enabled? */ static void setOcrOption(boolean enabled) { ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, OCR_ENABLED, (enabled ? "true" : "false")); //NON-NLS - } + } /** * Get OCR setting from permanent storage - * + * * @return Is OCR enabled? */ static boolean getOcrOption() { @@ -152,7 +154,7 @@ class KeywordSearchSettings { return OCR_ENABLED_DEFAULT; } } - + static void setShowSnippets(boolean showSnippets) { ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, SHOW_SNIPPETS, (showSnippets ? "true" : "false")); //NON-NLS } @@ -248,11 +250,41 @@ class KeywordSearchSettings { if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, OCR_ENABLED)) { logger.log(Level.INFO, "No configuration for OCR found, generating defaults..."); //NON-NLS KeywordSearchSettings.setOcrOption(OCR_ENABLED_DEFAULT); - } + } + //setting OCR default (disabled by default) + if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED)) { + logger.log(Level.INFO, "No configuration for OCR found, generating defaults..."); //NON-NLS + KeywordSearchSettings.setLimitedOcrOption(LIMITED_OCR_ENABLED_DEFAULT); + } //setting default Latin-1 Script if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_SCRIPTS, SCRIPT.LATIN_1.name())) { logger.log(Level.INFO, "No configuration for Scripts found, generating defaults..."); //NON-NLS ModuleSettings.setConfigSetting(KeywordSearchSettings.PROPERTIES_SCRIPTS, SCRIPT.LATIN_1.name(), Boolean.toString(true)); } } + + /** + * Enables the limiting OCR to be run on larger images and images which were + * extracted from documents. + * + * @param enabled Flag indicating if OCR is enabled. + */ + static void setLimitedOcrOption(boolean enabled) { + ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED, (enabled ? "true" : "false")); //NON-NLS + } + + /** + * Gets the limited OCR flag to indicate if OCR should be limited to larger + * images and images which were extracted from documents. + * + * @return Flag indicating if limited OCR is enabled. True if OCR should be + * limited, false otherwise.. + */ + static boolean getLimitedOcrOption() { + if (ModuleSettings.settingExists(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED)) { + return ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED).equals("true"); //NON-NLS + } else { + return LIMITED_OCR_ENABLED_DEFAULT; + } + } } diff --git a/NEWS.txt b/NEWS.txt index 532aae69fa..d9e2f99b90 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,3 +1,38 @@ +---------------- VERSION 4.19.0 -------------- +Data Source Management: +- To make managing big cases easier, all data sources are now associated with a host that can be specified in the “Add Data Source” wizard. +- Hosts can be grouped by “person”, which is simply a name of the owner. +- The main tree viewer can be configured to group by person and host. + +OS Accounts: +- Operating System (OS) accounts and realms are their own data types and not generic artifacts. +- OS Accounts are created for Windows accounts found in the registry. Domain-scoped realms are not fully detected yet. +- NTFS files are associated with OS Accounts by SID. +- The Recent Activity module associates artifacts with OS Accounts based on SID or path of database. Other modules still need to be updated. +- OS accounts appear in a dedicated sub-tree of the main tree view and their properties can be viewed in the results view. +- A new content viewer in the lower right area of the main window was built to display OS account data for the item selected in the result view. + +Discovery UI: +- Domain categorization and account types are displayed in Domain Discovery results. +- The Domain Discovery results view more explicitly shows when a downloaded file no longer exists. +- Check boxes are now used to select search options instead of shift-based multi-select. + +Ingest Modules: +- File metadata updates are batched up before being saved to the case database for better performance. +- Parsing of iLEAPP and aLEAPP output was expanded to create communication relationships which can be displayed in the Communications UI. +- EML email parsing handles EML messages that are attachments (and have their own attachments). +- Domain categorization within Recent Activity can be customized by user-defined rules that can be imported and exported. + +Miscellaneous: +- A “Reset Windows” feature was created to help redock windows. +- A case-insensitive wordlist of all words in the keyword search index can be exported as a text document. +- Information from the Data Source Summary panels can be exported as an Excel spreadsheet. +- More artifacts are added to the timeline and artifacts with multiple time-based attributes are mapped to multiple timeline events. +- The Auto Ingest Dashboard is resizable. +- Added option to only perform optical character recognition on certain file types. +- Heap dumps can be saved to a custom location. +- Assorted bug fixes are included. + ---------------- VERSION 4.18.0 -------------- Keyword Search: - A major upgrade from Solr 4 to Solr 8.6.3. Single user cases continue to use the embedded server. diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index 547b90ca6a..26181755d0 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -13,7 +13,6 @@ ChromeCacheExtractor.progressMsg={0}: Extracting cache entry {1} of {2} entries DataSourceUsage_AndroidMedia=Android Media Card DataSourceUsage_DJU_Drone_DAT=DJI Internal SD Card DataSourceUsage_FlashDrive=Flash Drive -# {0} - OS name DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0}) DataSourceUsageAnalyzer.parentModuleName=Recent Activity DomainCategoryRunner_moduleName_text=DomainCategoryRunner diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index e152a146fa..c2abe0223d 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -143,11 +143,27 @@ abstract class Extract { * @return The newly created artifact. */ BlackboardArtifact createArtifactWithAttributes(BlackboardArtifact.ARTIFACT_TYPE type, Content content, Collection attributes) throws TskCoreException { - Optional optional = getOsAccount(content); - if (optional.isPresent() && type.getCategory() == BlackboardArtifact.Category.DATA_ARTIFACT) { - return content.newDataArtifact(new BlackboardArtifact.Type(type), attributes, optional.get()); + return createArtifactWithAttributes(new BlackboardArtifact.Type(type), content, attributes); + } + + /** + * Generic method for creating artifacts. + * + * @param type The type of artifact. + * @param content The file the artifact originated from. + * @param attributes A list of the attributes to associate with the + * artifact. + * + * @return The newly created artifact. + * + * @throws TskCoreException + */ + BlackboardArtifact createArtifactWithAttributes(BlackboardArtifact.Type type, Content content, Collection attributes) throws TskCoreException { + Optional optional = getOsAccount(content); + if (optional.isPresent() && type.getCategory() == BlackboardArtifact.Category.DATA_ARTIFACT) { + return content.newDataArtifact(type, attributes, optional.get()); } else { - BlackboardArtifact bbart = content.newArtifact(type); + BlackboardArtifact bbart = content.newArtifact(type.getTypeID()); bbart.addAttributes(attributes); return bbart; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java index 4478f49278..1ced615661 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java @@ -441,7 +441,7 @@ final class ExtractRecycleBin extends Extract { attributes.add(new BlackboardAttribute(TSK_PATH, getName(), fileName)); attributes.add(new BlackboardAttribute(TSK_DATETIME_DELETED, getName(), dateTime)); attributes.add(new BlackboardAttribute(TSK_USER_NAME, getName(), userName == null || userName.isEmpty() ? "" : userName)); - return createArtifactWithAttributes(BlackboardArtifact.ARTIFACT_TYPE.fromID(type.getTypeID()), rFile, attributes); + return createArtifactWithAttributes(type, rFile, attributes); } /** diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 0c5cd988db..ad5c1e9ed4 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -93,10 +93,11 @@ import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.HostManager; import org.sleuthkit.datamodel.OsAccount; -import org.sleuthkit.datamodel.OsAccountAttribute; +import org.sleuthkit.datamodel.OsAccount.OsAccountAttribute; import org.sleuthkit.datamodel.OsAccountInstance; import org.sleuthkit.datamodel.OsAccountManager; import org.sleuthkit.datamodel.OsAccountManager.NotUserSIDException; +import org.sleuthkit.datamodel.OsAccountManager.OsAccountUpdateResult; import org.sleuthkit.datamodel.OsAccountRealm; import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamException; import org.sleuthkit.datamodel.Report; @@ -1079,7 +1080,7 @@ class ExtractRegistry extends Extract { // New OsAccount Code OsAccountManager accountMgr = tskCase.getOsAccountManager(); HostManager hostMrg = tskCase.getHostManager(); - Host host = hostMrg.getHost((DataSource)dataSource); + Host host = hostMrg.getHostByDataSource((DataSource)dataSource); List existingAccounts = accountMgr.getOsAccounts(host); for(OsAccount osAccount: existingAccounts) { @@ -1097,8 +1098,8 @@ class ExtractRegistry extends Extract { //add remaining userinfos as accounts; for (Map userInfo : userInfoMap.values()) { - OsAccount osAccount = accountMgr.createWindowsOsAccount(userInfo.get(SID_KEY), null, null, host, OsAccountRealm.RealmScope.UNKNOWN); - accountMgr.createOsAccountInstance(osAccount, (DataSource)dataSource, OsAccountInstance.OsAccountInstanceType.LAUNCHED); + OsAccount osAccount = accountMgr.newWindowsOsAccount(userInfo.get(SID_KEY), null, null, host, OsAccountRealm.RealmScope.UNKNOWN); + accountMgr.newOsAccountInstance(osAccount, (DataSource)dataSource, OsAccountInstance.OsAccountInstanceType.LAUNCHED); updateOsAccount(osAccount, userInfo, groupMap.get(userInfo.get(SID_KEY)), regAbstractFile); } @@ -1751,7 +1752,6 @@ class ExtractRegistry extends Extract { try { for (ShellBag bag : shellbags) { Collection attributes = new ArrayList<>(); - BlackboardArtifact artifact = regFile.newArtifact(getShellBagArtifact().getTypeID()); attributes.add(new BlackboardAttribute(TSK_PATH, getName(), bag.getResource())); attributes.add(new BlackboardAttribute(getKeyAttribute(), getName(), bag.getKey())); @@ -1776,9 +1776,7 @@ class ExtractRegistry extends Extract { attributes.add(new BlackboardAttribute(TSK_DATETIME_ACCESSED, getName(), time)); } - artifact.addAttributes(attributes); - - artifacts.add(artifact); + artifacts.add(createArtifactWithAttributes(getShellBagArtifact(), regFile, attributes)); } } finally { if(!context.dataSourceIngestIsCancelled()) { @@ -1967,17 +1965,18 @@ class ExtractRegistry extends Extract { private void createOrUpdateOsAccount(AbstractFile file, String sid, String userName, String homeDir) throws TskCoreException, TskDataException, NotUserSIDException { OsAccountManager accountMgr = tskCase.getOsAccountManager(); HostManager hostMrg = tskCase.getHostManager(); - Host host = hostMrg.getHost((DataSource)dataSource); + Host host = hostMrg.getHostByDataSource((DataSource)dataSource); Optional optional = accountMgr.getWindowsOsAccount(sid, null, null, host); OsAccount osAccount; if (!optional.isPresent()) { - osAccount = accountMgr.createWindowsOsAccount(sid, userName != null && userName.isEmpty() ? null : userName, null, host, OsAccountRealm.RealmScope.UNKNOWN); - accountMgr.createOsAccountInstance(osAccount, (DataSource)dataSource, OsAccountInstance.OsAccountInstanceType.LAUNCHED); + osAccount = accountMgr.newWindowsOsAccount(sid, userName != null && userName.isEmpty() ? null : userName, null, host, OsAccountRealm.RealmScope.UNKNOWN); + accountMgr.newOsAccountInstance(osAccount, (DataSource)dataSource, OsAccountInstance.OsAccountInstanceType.LAUNCHED); } else { osAccount = optional.get(); if (userName != null && !userName.isEmpty()) { - osAccount.setLoginName(userName); + OsAccountUpdateResult updateResult= accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, userName, null, host); + osAccount = updateResult.getUpdatedAccount().orElse(osAccount); } } @@ -1986,10 +1985,9 @@ class ExtractRegistry extends Extract { String dir = homeDir.replaceFirst("^(%\\w*%)", ""); dir = dir.replace("\\", "/"); attributes.add(createOsAccountAttribute(TSK_HOME_DIR, dir, osAccount, host, file)); - accountMgr.addOsAccountAttributes(osAccount, attributes); + accountMgr.addExtendedOsAccountAttributes(osAccount, attributes); } - accountMgr.updateOsAccount(osAccount); } /** @@ -2040,7 +2038,7 @@ class ExtractRegistry extends Extract { * @throws TskDataException * @throws TskCoreException */ - private void updateOsAccount(OsAccount osAccount, Map userInfo, List groupList, AbstractFile regFile) throws TskDataException, TskCoreException { + private void updateOsAccount(OsAccount osAccount, Map userInfo, List groupList, AbstractFile regFile) throws TskDataException, TskCoreException, NotUserSIDException { Host host = ((DataSource)dataSource).getHost(); SimpleDateFormat regRipperTimeFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy 'Z'", US); @@ -2048,12 +2046,11 @@ class ExtractRegistry extends Extract { List attributes = new ArrayList<>(); + Long creationTime = null; + String value = userInfo.get(ACCOUNT_CREATED_KEY); if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { - Long time = parseRegRipTime(value); - if (time != null) { - osAccount.setCreationTime(time); - } + creationTime = parseRegRipTime(value); } value = userInfo.get(LAST_LOGIN_KEY); @@ -2066,9 +2063,10 @@ class ExtractRegistry extends Extract { } } + String loginName = null; value = userInfo.get(USERNAME_KEY); if (value != null && !value.isEmpty()) { - osAccount.setLoginName(value); + loginName = value; } value = userInfo.get(LOGIN_COUNT_KEY); @@ -2102,13 +2100,14 @@ class ExtractRegistry extends Extract { } // FULL_NAME_KEY and NAME_KEY appear to be the same value. + String fullName = null; value = userInfo.get(FULL_NAME_KEY); if (value != null && !value.isEmpty()) { - osAccount.setFullName(value); + fullName = value; } else { value = userInfo.get(NAME_KEY); if (value != null && !value.isEmpty()) { - osAccount.setFullName(value); + fullName = value; } } @@ -2163,9 +2162,17 @@ class ExtractRegistry extends Extract { groups, osAccount, host, regFile)); } + // add the attributes to account. OsAccountManager accountMgr = tskCase.getOsAccountManager(); - accountMgr.addOsAccountAttributes(osAccount, attributes); - accountMgr.updateOsAccount(osAccount); + accountMgr.addExtendedOsAccountAttributes(osAccount, attributes); + + // update the loginname + accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, loginName, null, host); + + // update other standard attributes - fullname, creationdate + accountMgr.updateStandardOsAccountAttributes(osAccount, fullName, null, null, creationTime); + + } /** @@ -2205,7 +2212,7 @@ class ExtractRegistry extends Extract { * @return Newly created OsACcountAttribute */ private OsAccountAttribute createOsAccountAttribute(BlackboardAttribute.ATTRIBUTE_TYPE type, String value, OsAccount osAccount, Host host, AbstractFile file) { - return new OsAccountAttribute(new BlackboardAttribute.Type(type), value, osAccount, host, file); + return osAccount.new OsAccountAttribute(new BlackboardAttribute.Type(type), value, osAccount, host, file); } /** @@ -2220,7 +2227,7 @@ class ExtractRegistry extends Extract { * @return Newly created OsACcountAttribute */ private OsAccountAttribute createOsAccountAttribute(BlackboardAttribute.ATTRIBUTE_TYPE type, Long value, OsAccount osAccount, Host host, AbstractFile file) { - return new OsAccountAttribute(new BlackboardAttribute.Type(type), value, osAccount, host, file); + return osAccount.new OsAccountAttribute(new BlackboardAttribute.Type(type), value, osAccount, host, file); } /** @@ -2235,6 +2242,6 @@ class ExtractRegistry extends Extract { * @return Newly created OsACcountAttribute */ private OsAccountAttribute createOsAccountAttribute(BlackboardAttribute.ATTRIBUTE_TYPE type, Integer value, OsAccount osAccount, Host host, AbstractFile file) { - return new OsAccountAttribute(new BlackboardAttribute.Type(type), value, osAccount, host, file); + return osAccount.new OsAccountAttribute(new BlackboardAttribute.Type(type), value, osAccount, host, file); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java index 788f219371..c1206c82f6 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAOsAccountCache.java @@ -29,7 +29,7 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.OsAccount; -import org.sleuthkit.datamodel.OsAccountAttribute; +import org.sleuthkit.datamodel.OsAccount.OsAccountAttribute; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -117,7 +117,7 @@ final class RAOsAccountCache { List accounts = tskCase.getOsAccountManager().getOsAccounts(host); for (OsAccount account : accounts) { - List attributeList = account.getOsAccountAttributes(); + List attributeList = account.getExtendedOsAccountAttributes(); for (OsAccountAttribute attribute : attributeList) { if (attribute.getHostId().isPresent() diff --git a/thirdparty/aLeapp/aleapp.exe b/thirdparty/aLeapp/aleapp.exe index 71106090c0..3f47725616 100644 Binary files a/thirdparty/aLeapp/aleapp.exe and b/thirdparty/aLeapp/aleapp.exe differ diff --git a/thirdparty/iLeapp/ileapp.exe b/thirdparty/iLeapp/ileapp.exe index 0456c3c726..f1d395e930 100644 Binary files a/thirdparty/iLeapp/ileapp.exe and b/thirdparty/iLeapp/ileapp.exe differ