mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into timeline-event-mgr

Browse Source 
# Conflicts:
#	Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java
#	Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java
#	Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java
#	RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED
#	RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java
#	RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java
#	RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java
#	RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java
This commit is contained in:
millmanorama 2019-03-06 16:32:42 +01:00
commit ce6807b96e
61 changed files with 5694 additions and 1089 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Core/ivy.xml
View File

@ -30,7 +30,6 @@
<dependency conf="core->default" org="org.apache.commons" name="commons-pool2" rev="2.4.2"/>
<dependency conf="core->default" org="org.jsoup" name="jsoup" rev="1.10.3"/>
<dependency conf="core->default" org="com.googlecode.plist" name="dd-plist" rev="1.20"/>
<dependency conf="core->default" org="com.fasterxml.jackson.core" name="jackson-core" rev="2.9.7"/>

1
Core/nbproject/project.properties
View File

@ -12,7 +12,6 @@ file.reference.commons-compress-1.14.jar=release/modules/ext/commons-compress-1.
file.reference.commons-dbcp2-2.1.1.jar=release/modules/ext/commons-dbcp2-2.1.1.jar
file.reference.commons-io-2.5.jar=release/modules/ext/commons-io-2.5.jar
file.reference.commons-pool2-2.4.2.jar=release/modules/ext/commons-pool2-2.4.2.jar
file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar
file.reference.geoapi-3.0.0.jar=release/modules/ext/geoapi-3.0.0.jar
file.reference.grib-4.5.5.jar=release/modules/ext/grib-4.5.5.jar
file.reference.httpservices-4.5.5.jar=release/modules/ext/httpservices-4.5.5.jar

4
Core/nbproject/project.xml
View File

@ -629,10 +629,6 @@
<runtime-relative-path>ext/Rejistry-1.0-SNAPSHOT.jar</runtime-relative-path>
<binary-origin>release/modules/ext/Rejistry-1.0-SNAPSHOT.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/dd-plist-1.20.jar</runtime-relative-path>
<binary-origin>release/modules/ext/dd-plist-1.20.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/rome-1.5.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/rome-1.5.1.jar</binary-origin>

1
Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties
View File

@ -6,3 +6,4 @@ DataContentViewerOtherCases.showCommonalityMenuItem.text=Show Frequency
DataContentViewerOtherCases.earliestCaseDate.text=Earliest Case Date
DataContentViewerOtherCases.earliestCaseLabel.toolTipText=
DataContentViewerOtherCases.earliestCaseLabel.text=Central Repository Starting Date:
DataContentViewerOtherCases.foundInLabel.text=

2
Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties-MERGED
View File

@ -10,6 +10,7 @@ DataContentViewerOtherCases.correlatedArtifacts.failed=Failed to get frequency d
DataContentViewerOtherCases.correlatedArtifacts.isEmpty=There are no files or artifacts to correlate.
DataContentViewerOtherCases.correlatedArtifacts.title=Attribute Frequency
DataContentViewerOtherCases.earliestCaseNotAvailable=\ Not Enabled.
DataContentViewerOtherCases.foundIn.text=Found %d instances in %d cases and %d data sources.
DataContentViewerOtherCases.noOpenCase.errMsg=No open case available.
DataContentViewerOtherCases.selectAllMenuItem.text=Select All
DataContentViewerOtherCases.showCaseDetailsMenuItem.text=Show Case Details
@ -22,6 +23,7 @@ DataContentViewerOtherCases.showCommonalityMenuItem.text=Show Frequency
DataContentViewerOtherCases.earliestCaseDate.text=Earliest Case Date
DataContentViewerOtherCases.earliestCaseLabel.toolTipText=
DataContentViewerOtherCases.earliestCaseLabel.text=Central Repository Starting Date:
DataContentViewerOtherCases.foundInLabel.text=
DataContentViewerOtherCases.title=Other Occurrences
DataContentViewerOtherCases.toolTip=Displays instances of the selected file/artifact from other occurrences.
DataContentViewerOtherCasesTableModel.attribute=Matched Attribute

48
Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.form
View File

@ -73,7 +73,7 @@
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="otherCasesPanel" pref="58" max="32767" attributes="0"/>
<Component id="otherCasesPanel" pref="53" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
</Layout>
@ -96,10 +96,10 @@
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<EmptySpace min="0" pref="483" max="32767" attributes="0"/>
<EmptySpace min="0" pref="61" max="32767" attributes="0"/>
<Group type="103" rootIndex="1" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<Component id="tableContainerPanel" pref="58" max="32767" attributes="0"/>
<Component id="tableContainerPanel" pref="53" max="32767" attributes="0"/>
<EmptySpace min="-2" pref="0" max="-2" attributes="0"/>
</Group>
</Group>
@ -117,31 +117,28 @@
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="1" attributes="0">
<Component id="tableStatusPanel" pref="1282" max="32767" attributes="0"/>
<EmptySpace min="-2" pref="218" max="-2" attributes="0"/>
</Group>
<Component id="tableScrollPane" alignment="0" max="32767" attributes="0"/>
<Component id="tableScrollPane" alignment="0" pref="1508" max="32767" attributes="0"/>
<Group type="102" alignment="0" attributes="0">
<Component id="earliestCaseLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="earliestCaseDate" min="-2" max="-2" attributes="0"/>
<EmptySpace max="32767" attributes="0"/>
<EmptySpace min="-2" pref="66" max="-2" attributes="0"/>
<Component id="foundInLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace min="0" pref="1157" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="1" attributes="0">
<Component id="tableScrollPane" pref="27" max="32767" attributes="0"/>
<EmptySpace min="-2" pref="2" max="-2" attributes="0"/>
<Component id="tableScrollPane" pref="71" max="32767" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="earliestCaseLabel" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="earliestCaseDate" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="foundInLabel" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace min="0" pref="0" max="-2" attributes="0"/>
<Component id="tableStatusPanel" min="-2" max="-2" attributes="0"/>
<EmptySpace min="0" pref="0" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="6" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
@ -198,26 +195,13 @@
</Property>
</Properties>
</Component>
<Container class="javax.swing.JPanel" name="tableStatusPanel">
<Component class="javax.swing.JLabel" name="foundInLabel">
<Properties>
<Property name="preferredSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="[1500, 16]"/>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties" key="DataContentViewerOtherCases.foundInLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<EmptySpace min="0" pref="0" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<EmptySpace min="0" pref="16" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
</Layout>
</Container>
</Component>
</SubComponents>
</Container>
</SubComponents>

86
Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java
View File

@ -1,7 +1,7 @@
/*
* Central Repository
*
* Copyright 2015-2018 Basis Technology Corp.
* Copyright 2015-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -33,10 +33,12 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import javax.swing.JFileChooser;
@ -323,6 +325,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
tableModel.clearTable();
correlationAttributes.clear();
earliestCaseDate.setText(Bundle.DataContentViewerOtherCases_earliestCaseNotAvailable());
foundInLabel.setText("");
}
@Override
@ -354,6 +357,40 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
public int isPreferred(Node node) {
return 1;
}
/**
* Set the number of unique cases and data sources.
*/
@Messages({
"DataContentViewerOtherCases.foundIn.text=Found %d instances in %d cases and %d data sources."
})
private void setOccurrenceCounts() {
DataContentViewerOtherCasesTableModel model = (DataContentViewerOtherCasesTableModel) otherCasesTable.getModel();
int caseColumnIndex = DataContentViewerOtherCasesTableModel.TableColumns.CASE_NAME.ordinal();
int deviceColumnIndex = DataContentViewerOtherCasesTableModel.TableColumns.DEVICE.ordinal();
/*
* We need a unique set of data sources. We rely on device ID for this.
* To mitigate edge cases where a device ID could be duplicated in the
* same case (e.g. "report.xml"), we put the device ID and case name in
* a key-value pair.
*
* Note: Relying on the case name isn't a fool-proof way of determining
* a case to be unique. We should improve this in the future.
*/
Set<String> cases = new HashSet<>();
Map<String, String> devices = new HashMap<>();
for (int i=0; i < model.getRowCount(); i++) {
String caseName = (String) model.getValueAt(i, caseColumnIndex);
String deviceId = (String) model.getValueAt(i, deviceColumnIndex);
cases.add(caseName);
devices.put(deviceId, caseName);
}
foundInLabel.setText(String.format(Bundle.DataContentViewerOtherCases_foundIn_text(), model.getRowCount(), cases.size(), devices.size()));
}
/**
* Get the associated BlackboardArtifact from a node, if it exists.
@ -737,7 +774,9 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
} else {
setColumnWidths();
}
setEarliestCaseDate();
setOccurrenceCounts();
}
/**
@ -789,7 +828,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
otherCasesTable = new javax.swing.JTable();
earliestCaseLabel = new javax.swing.JLabel();
earliestCaseDate = new javax.swing.JLabel();
tableStatusPanel = new javax.swing.JPanel();
foundInLabel = new javax.swing.JLabel();
rightClickPopupMenu.addPopupMenuListener(new javax.swing.event.PopupMenuListener() {
public void popupMenuCanceled(javax.swing.event.PopupMenuEvent evt) {
@ -835,44 +874,31 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
org.openide.awt.Mnemonics.setLocalizedText(earliestCaseDate, org.openide.util.NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.earliestCaseDate.text")); // NOI18N
tableStatusPanel.setPreferredSize(new java.awt.Dimension(1500, 16));
javax.swing.GroupLayout tableStatusPanelLayout = new javax.swing.GroupLayout(tableStatusPanel);
tableStatusPanel.setLayout(tableStatusPanelLayout);
tableStatusPanelLayout.setHorizontalGroup(
tableStatusPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGap(0, 0, Short.MAX_VALUE)
);
tableStatusPanelLayout.setVerticalGroup(
tableStatusPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGap(0, 16, Short.MAX_VALUE)
);
org.openide.awt.Mnemonics.setLocalizedText(foundInLabel, org.openide.util.NbBundle.getMessage(DataContentViewerOtherCases.class, "DataContentViewerOtherCases.foundInLabel.text")); // NOI18N
javax.swing.GroupLayout tableContainerPanelLayout = new javax.swing.GroupLayout(tableContainerPanel);
tableContainerPanel.setLayout(tableContainerPanelLayout);
tableContainerPanelLayout.setHorizontalGroup(
tableContainerPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(javax.swing.GroupLayout.Alignment.TRAILING, tableContainerPanelLayout.createSequentialGroup()
.addComponent(tableStatusPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 1282, Short.MAX_VALUE)
.addGap(218, 218, 218))
.addComponent(tableScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(tableScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 1508, Short.MAX_VALUE)
.addGroup(tableContainerPanelLayout.createSequentialGroup()
.addComponent(earliestCaseLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(earliestCaseDate)
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addGap(66, 66, 66)
.addComponent(foundInLabel)
.addGap(0, 1157, Short.MAX_VALUE))
);
tableContainerPanelLayout.setVerticalGroup(
tableContainerPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(javax.swing.GroupLayout.Alignment.TRAILING, tableContainerPanelLayout.createSequentialGroup()
.addComponent(tableScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 27, Short.MAX_VALUE)
.addGap(2, 2, 2)
.addComponent(tableScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 71, Short.MAX_VALUE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(tableContainerPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(earliestCaseLabel)
.addComponent(earliestCaseDate))
.addGap(0, 0, 0)
.addComponent(tableStatusPanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addGap(0, 0, 0))
.addComponent(earliestCaseDate)
.addComponent(foundInLabel))
.addGap(6, 6, 6))
);
javax.swing.GroupLayout otherCasesPanelLayout = new javax.swing.GroupLayout(otherCasesPanel);
@ -885,10 +911,10 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
);
otherCasesPanelLayout.setVerticalGroup(
otherCasesPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGap(0, 483, Short.MAX_VALUE)
.addGap(0, 61, Short.MAX_VALUE)
.addGroup(otherCasesPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(otherCasesPanelLayout.createSequentialGroup()
.addComponent(tableContainerPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 59, Short.MAX_VALUE)
.addComponent(tableContainerPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 53, Short.MAX_VALUE)
.addGap(0, 0, 0)))
);
@ -900,7 +926,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(otherCasesPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 59, Short.MAX_VALUE)
.addComponent(otherCasesPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 53, Short.MAX_VALUE)
);
}// </editor-fold>//GEN-END:initComponents
@ -924,6 +950,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
private javax.swing.JLabel earliestCaseDate;
private javax.swing.JLabel earliestCaseLabel;
private javax.swing.JMenuItem exportToCSVMenuItem;
private javax.swing.JLabel foundInLabel;
private javax.swing.JPanel otherCasesPanel;
private javax.swing.JTable otherCasesTable;
private javax.swing.JPopupMenu rightClickPopupMenu;
@ -932,7 +959,6 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi
private javax.swing.JMenuItem showCommonalityMenuItem;
private javax.swing.JPanel tableContainerPanel;
private javax.swing.JScrollPane tableScrollPane;
private javax.swing.JPanel tableStatusPanel;
// End of variables declaration//GEN-END:variables
/**

137
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java
View File

@ -34,6 +34,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.time.LocalDate;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@ -61,7 +62,7 @@ abstract class AbstractSqlEamDb implements EamDb {
static final String SCHEMA_MINOR_VERSION_KEY = "SCHEMA_MINOR_VERSION";
static final String CREATION_SCHEMA_MAJOR_VERSION_KEY = "CREATION_SCHEMA_MAJOR_VERSION";
static final String CREATION_SCHEMA_MINOR_VERSION_KEY = "CREATION_SCHEMA_MINOR_VERSION";
static final CaseDbSchemaVersionNumber SOFTWARE_CR_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 2);
static final CaseDbSchemaVersionNumber SOFTWARE_CR_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 3);
protected final List<CorrelationAttributeInstance.Type> defaultCorrelationTypes;
@ -105,6 +106,11 @@ abstract class AbstractSqlEamDb implements EamDb {
});
}
/**
* Setup and create a connection to the selected database implementation
*/
protected abstract Connection connect(boolean foreignKeys) throws EamDbException;
/**
* Setup and create a connection to the selected database implementation
*/
@ -625,7 +631,7 @@ abstract class AbstractSqlEamDb implements EamDb {
// This data source is already in the central repo
return eamDataSource;
}
Connection conn = connect();
PreparedStatement preparedStatement = null;
@ -650,7 +656,7 @@ abstract class AbstractSqlEamDb implements EamDb {
/*
* If nothing was inserted, then return the data source that
* exists in the Central Repository.
*
*
* This is expected to occur with PostgreSQL Central Repository
* databases.
*/
@ -675,7 +681,7 @@ abstract class AbstractSqlEamDb implements EamDb {
* If an exception was thrown causing us to not return a new data
* source, attempt to get an existing data source with the same case
* ID and data source object ID.
*
*
* This exception block is expected to occur with SQLite Central
* Repository databases.
*/
@ -1052,30 +1058,43 @@ abstract class AbstractSqlEamDb implements EamDb {
}
}
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValue of the given eamArtifact.
*
* @param aType The type of the artifact
* @param value The correlation value
*
* @return List of artifact instances for a given type/value
*
* @throws EamDbException
*/
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException {
return getArtifactInstancesByTypeValues(aType, Arrays.asList(value));
}
String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value);
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List<String> values) throws EamDbException, CorrelationAttributeNormalizationException {
return getArtifactInstances(prepareGetInstancesSql(aType, values), aType);
}
Connection conn = connect();
List<CorrelationAttributeInstance> artifactInstances = new ArrayList<>();
CorrelationAttributeInstance artifactInstance;
PreparedStatement preparedStatement = null;
ResultSet resultSet = null;
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List<String> values, List<Integer> caseIds) throws EamDbException, CorrelationAttributeNormalizationException {
String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType);
String sql
= " and "
+ tableName
+ ".case_id in ('";
StringBuilder inValuesBuilder = new StringBuilder(prepareGetInstancesSql(aType, values));
inValuesBuilder.append(sql);
inValuesBuilder.append(caseIds.stream().map(String::valueOf).collect(Collectors.joining("', '")));
inValuesBuilder.append("')");
return getArtifactInstances(inValuesBuilder.toString(), aType);
}
/**
* Get the select statement for retrieving correlation attribute instances
* from the CR for a given type with values matching the specified values
*
* @param aType The type of the artifact
* @param values The list of correlation values to get
* CorrelationAttributeInstances for
*
* @return the select statement as a String
*
* @throws CorrelationAttributeNormalizationException
*/
private String prepareGetInstancesSql(CorrelationAttributeInstance.Type aType, List<String> values) throws CorrelationAttributeNormalizationException {
String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType);
String sql
= "SELECT "
@ -1093,11 +1112,42 @@ abstract class AbstractSqlEamDb implements EamDb {
+ " LEFT JOIN data_sources ON "
+ tableName
+ ".data_source_id=data_sources.id"
+ " WHERE value=?";
+ " WHERE value IN (";
StringBuilder inValuesBuilder = new StringBuilder(sql);
for (String value : values) {
if (value != null) {
inValuesBuilder.append("'");
inValuesBuilder.append(CorrelationAttributeNormalizer.normalize(aType, value));
inValuesBuilder.append("',");
}
}
inValuesBuilder.deleteCharAt(inValuesBuilder.length() - 1); //delete last comma
inValuesBuilder.append(")");
return inValuesBuilder.toString();
}
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValues of the given eamArtifact.
*
* @param aType The type of the artifact
* @param values The list of correlation values to get
* CorrelationAttributeInstances for
*
* @return List of artifact instances for a given type with the specified
* values
*
* @throws CorrelationAttributeNormalizationException
* @throws EamDbException
*/
private List<CorrelationAttributeInstance> getArtifactInstances(String sql, CorrelationAttributeInstance.Type aType) throws CorrelationAttributeNormalizationException, EamDbException {
Connection conn = connect();
List<CorrelationAttributeInstance> artifactInstances = new ArrayList<>();
CorrelationAttributeInstance artifactInstance;
PreparedStatement preparedStatement = null;
ResultSet resultSet = null;
try {
preparedStatement = conn.prepareStatement(sql);
preparedStatement.setString(1, normalizedValue);
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
artifactInstance = getEamArtifactInstanceFromResultSet(resultSet, aType);
@ -1110,7 +1160,6 @@ abstract class AbstractSqlEamDb implements EamDb {
EamDbUtil.closeResultSet(resultSet);
EamDbUtil.closeConnection(conn);
}
return artifactInstances;
}
@ -3338,12 +3387,13 @@ abstract class AbstractSqlEamDb implements EamDb {
Statement statement = null;
PreparedStatement preparedStatement = null;
Connection conn = null;
EamDbPlatformEnum selectedPlatform = null;
try {
conn = connect();
conn = connect(false);
conn.setAutoCommit(false);
statement = conn.createStatement();
selectedPlatform = EamDbPlatformEnum.getSelectedPlatform();
int minorVersion = 0;
String minorVersionStr = null;
resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='" + AbstractSqlEamDb.SCHEMA_MINOR_VERSION_KEY + "'");
@ -3397,8 +3447,6 @@ abstract class AbstractSqlEamDb implements EamDb {
return;
}
EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform();
/*
* Update to 1.1
*/
@ -3582,7 +3630,36 @@ abstract class AbstractSqlEamDb implements EamDb {
statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MAJOR_VERSION_KEY + "','" + creationMajorVer + "')");
statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MINOR_VERSION_KEY + "','" + creationMinorVer + "')");
}
/*
* Update to 1.3
*/
if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 3)) < 0) {
switch (selectedPlatform) {
case POSTGRESQL:
statement.execute("ALTER TABLE data_sources DROP CONSTRAINT datasource_unique");
//unique constraint for upgraded data_sources table is purposefully different than new data_sources table
statement.execute("ALTER TABLE data_sources ADD CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id)");
break;
case SQLITE:
statement.execute("DROP INDEX IF EXISTS data_sources_name");
statement.execute("DROP INDEX IF EXISTS data_sources_object_id");
statement.execute("ALTER TABLE data_sources RENAME TO old_data_sources");
//unique constraint for upgraded data_sources table is purposefully different than new data_sources table
statement.execute("CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL,"
+ "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer,"
+ "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL,"
+ "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
+ "CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name, datasource_obj_id))");
statement.execute(SqliteEamDbSettings.getAddDataSourcesNameIndexStatement());
statement.execute(SqliteEamDbSettings.getAddDataSourcesObjectIdIndexStatement());
statement.execute("INSERT INTO data_sources SELECT * FROM old_data_sources");
statement.execute("DROP TABLE old_data_sources");
break;
default:
throw new EamDbException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded.");
}
}
updateSchemaVersion(conn);
conn.commit();
logger.log(Level.INFO, String.format("Central Repository schema updated to version %s", SOFTWARE_CR_DB_SCHEMA_VERSION));

71
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java
View File

@ -24,7 +24,6 @@ import java.util.Set;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService;
import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber;
/**
* Main interface for interacting with the database
@ -200,27 +199,29 @@ public interface EamDb {
* Creates new Data Source in the database
*
* @param eamDataSource the data source to add
*
* @return - A CorrelationDataSource object with data source's central repository id
*
* @return - A CorrelationDataSource object with data source's central
* repository id
*/
CorrelationDataSource newDataSource(CorrelationDataSource eamDataSource) throws EamDbException;
/**
* Updates the MD5 hash value in an existing data source in the database.
*
* @param eamDataSource The data source to update
*/
void updateDataSourceMd5Hash(CorrelationDataSource eamDataSource) throws EamDbException;
/**
* Updates the SHA-1 hash value in an existing data source in the database.
*
* @param eamDataSource The data source to update
*/
void updateDataSourceSha1Hash(CorrelationDataSource eamDataSource) throws EamDbException;
/**
* Updates the SHA-256 hash value in an existing data source in the database.
* Updates the SHA-256 hash value in an existing data source in the
* database.
*
* @param eamDataSource The data source to update
*/
@ -257,14 +258,14 @@ public interface EamDb {
/**
* Changes the name of a data source in the DB
*
* @param eamDataSource The data source
* @param newName The new name
*
* @throws EamDbException
*
* @param eamDataSource The data source
* @param newName The new name
*
* @throws EamDbException
*/
void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException;
/**
* Inserts new Artifact(s) into the database. Should add associated Case and
* Data Source first.
@ -273,17 +274,55 @@ public interface EamDb {
*/
void addArtifactInstance(CorrelationAttributeInstance eamArtifact) throws EamDbException;
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValues of the given eamArtifact.
*
* @param aType EamArtifact.Type to search for
* @param values The list of correlation values to get
* CorrelationAttributeInstances for
*
* @return List of artifact instances for a given type with the specified
* values
*
* @throws CorrelationAttributeNormalizationException
* @throws EamDbException
*/
List<CorrelationAttributeInstance> getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List<String> values) throws EamDbException, CorrelationAttributeNormalizationException;
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValue of the given eamArtifact.
*
* @param aType EamArtifact.Type to search for
* @param value Value to search for
* @param aType The type of the artifact
* @param value The correlation value
*
* @return List of artifact instances for a given type/value
*
* @throws CorrelationAttributeNormalizationException
* @throws EamDbException
*/
List<CorrelationAttributeInstance> getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException;
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValues of the given eamArtifact
* for the specified cases.
*
* @param aType The type of the artifact
* @param values The list of correlation values to get
* CorrelationAttributeInstances for
* @param caseIds The list of central repository case ids to get
* CorrelationAttributeInstances for
*
* @return List of artifact instances for a given type with the specified
* values for the specified cases
*
* @throws CorrelationAttributeNormalizationException
* @throws EamDbException
*/
List<CorrelationAttributeInstance> getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List<String> values, List<Integer> caseIds) throws EamDbException, CorrelationAttributeNormalizationException;
/**
* Retrieves eamArtifact instances from the database that are associated
* with the aType and filePath
@ -341,7 +380,7 @@ public interface EamDb {
* Retrieves number of eamArtifact instances in the database that are
* associated with the given data source.
*
* @param correlationDataSource Data source to search for
* @param correlationDataSource Data source to search for
*
* @return Number of artifact instances having caseDisplayName and
* dataSource

16
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java
View File

@ -161,6 +161,21 @@ final class PostgresEamDb extends AbstractSqlEamDb {
connectionPool.setValidationQuery(dbSettings.getValidationQuery());
}
/**
* Lazily setup Singleton connection on first request.
*
* @param foreignKeys -ignored arguement with postgres databases
*
* @return A connection from the connection pool.
*
* @throws EamDbException
*/
@Override
protected Connection connect(boolean foreignKeys) throws EamDbException {
//foreignKeys boolean is ignored for postgres
return connect();
}
/**
* Lazily setup Singleton connection on first request.
*
@ -179,7 +194,6 @@ final class PostgresEamDb extends AbstractSqlEamDb {
setupConnectionPool();
}
}
try {
return connectionPool.getConnection();
} catch (SQLException ex) {

110
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java
View File

@ -1,7 +1,7 @@
/*
* Central Repository
*
* Copyright 2015-2017 Basis Technology Corp.
* Copyright 2015-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -43,16 +43,16 @@ import static org.sleuthkit.autopsy.centralrepository.datamodel.AbstractSqlEamDb
public final class PostgresEamDbSettings {
private final static Logger LOGGER = Logger.getLogger(PostgresEamDbSettings.class.getName());
private final String DEFAULT_HOST = ""; // NON-NLS
private final int DEFAULT_PORT = 5432;
private final String DEFAULT_DBNAME = "central_repository"; // NON-NLS
private final String DEFAULT_USERNAME = "";
private final String DEFAULT_PASSWORD = "";
private final String VALIDATION_QUERY = "SELECT version()"; // NON-NLS
private final String JDBC_BASE_URI = "jdbc:postgresql://"; // NON-NLS
private final String JDBC_DRIVER = "org.postgresql.Driver"; // NON-NLS
private final String DB_NAMES_REGEX = "[a-z][a-z0-9_]*"; // only lower case
private final String DB_USER_NAMES_REGEX = "[a-zA-Z]\\w*";
private final static String DEFAULT_HOST = ""; // NON-NLS
private final static int DEFAULT_PORT = 5432;
private final static String DEFAULT_DBNAME = "central_repository"; // NON-NLS
private final static String DEFAULT_USERNAME = "";
private final static String DEFAULT_PASSWORD = "";
private final static String VALIDATION_QUERY = "SELECT version()"; // NON-NLS
private final static String JDBC_BASE_URI = "jdbc:postgresql://"; // NON-NLS
private final static String JDBC_DRIVER = "org.postgresql.Driver"; // NON-NLS
private final static String DB_NAMES_REGEX = "[a-z][a-z0-9_]*"; // only lower case
private final static String DB_USER_NAMES_REGEX = "[a-zA-Z]\\w*";
private String host;
private int port;
private String dbName;
@ -339,23 +339,6 @@ public final class PostgresEamDbSettings {
String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)";
String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)";
StringBuilder createDataSourcesTable = new StringBuilder();
createDataSourcesTable.append("CREATE TABLE IF NOT EXISTS data_sources (");
createDataSourcesTable.append("id SERIAL PRIMARY KEY,");
createDataSourcesTable.append("case_id integer NOT NULL,");
createDataSourcesTable.append("device_id text NOT NULL,");
createDataSourcesTable.append("name text NOT NULL,");
createDataSourcesTable.append("datasource_obj_id BIGINT,");
createDataSourcesTable.append("md5 text DEFAULT NULL,");
createDataSourcesTable.append("sha1 text DEFAULT NULL,");
createDataSourcesTable.append("sha256 text DEFAULT NULL,");
createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name)");
createDataSourcesTable.append(")");
String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)";
String dataSourceIdx2 = "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)";
StringBuilder createReferenceSetsTable = new StringBuilder();
createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets (");
createReferenceSetsTable.append("id SERIAL PRIMARY KEY,");
@ -422,10 +405,10 @@ public final class PostgresEamDbSettings {
stmt.execute(casesIdx1);
stmt.execute(casesIdx2);
stmt.execute(createDataSourcesTable.toString());
stmt.execute(dataSourceIdx1);
stmt.execute(dataSourceIdx2);
stmt.execute(getCreateDataSourcesTableStatement());
stmt.execute(getAddDataSourcesNameIndexStatement());
stmt.execute(getAddDataSourcesObjectIdIndexStatement());
stmt.execute(createReferenceSetsTable.toString());
stmt.execute(referenceSetsIdx1);
@ -487,21 +470,50 @@ public final class PostgresEamDbSettings {
*/
static String getCreateArtifactInstancesTableTemplate() {
// Each "%s" will be replaced with the relevant TYPE_instances table name.
StringBuilder createArtifactInstancesTableTemplate = new StringBuilder();
createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s (");
createArtifactInstancesTableTemplate.append("id SERIAL PRIMARY KEY,");
createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,");
createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,");
createArtifactInstancesTableTemplate.append("value text NOT NULL,");
createArtifactInstancesTableTemplate.append("file_path text NOT NULL,");
createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,");
createArtifactInstancesTableTemplate.append("comment text,");
createArtifactInstancesTableTemplate.append("file_obj_id BIGINT,");
createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),");
createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL");
createArtifactInstancesTableTemplate.append(")");
return createArtifactInstancesTableTemplate.toString();
return ("CREATE TABLE IF NOT EXISTS %s (id SERIAL PRIMARY KEY,case_id integer NOT NULL,"
+ "data_source_id integer NOT NULL,value text NOT NULL,file_path text NOT NULL,"
+ "known_status integer NOT NULL,comment text,file_obj_id BIGINT,"
+ "CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"
+ "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
+ "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)");
}
/**
* Get the statement String for creating a new data_sources table in a
* Postgres central repository.
*
* @return a String which is a statement for cretating a new data_sources
* table
*/
static String getCreateDataSourcesTableStatement() {
return "CREATE TABLE IF NOT EXISTS data_sources "
+ "(id SERIAL PRIMARY KEY,case_id integer NOT NULL,device_id text NOT NULL,"
+ "name text NOT NULL,datasource_obj_id BIGINT,md5 text DEFAULT NULL,"
+ "sha1 text DEFAULT NULL,sha256 text DEFAULT NULL,"
+ "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
+ "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))";
}
/**
* Get the statement for creating an index on the name column of the
* data_sources table.
*
* @return a String which is a statement for adding an index on the name
* column of the data_sources table.
*/
static String getAddDataSourcesNameIndexStatement() {
return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)";
}
/**
* Get the statement for creating an index on the data_sources_object_id
* column of the data_sources table.
*
* @return a String which is a statement for adding an index on the
* data_sources_object_id column of the data_sources table.
*/
static String getAddDataSourcesObjectIdIndexStatement() {
return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)";
}
/**
@ -561,8 +573,8 @@ public final class PostgresEamDbSettings {
* instance table. %s will exist in the template where the name of the new
* table will be addedd.
*
* @return a String which is a template for adding an index to the file_obj_id
* column of a _instances table
* @return a String which is a template for adding an index to the
* file_obj_id column of a _instances table
*/
static String getAddObjectIdIndexTemplate() {
// Each "%s" will be replaced with the relevant TYPE_instances table name.

80
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java
View File

@ -153,7 +153,7 @@ final class SqliteEamDb extends AbstractSqlEamDb {
* Setup a connection pool for db connections.
*
*/
private void setupConnectionPool() throws EamDbException {
private void setupConnectionPool(boolean foreignKeysEnabled) throws EamDbException {
if (dbSettings.dbFileExists() == false) {
throw new EamDbException("Central repository database missing");
@ -169,27 +169,31 @@ final class SqliteEamDb extends AbstractSqlEamDb {
connectionPool.setMaxIdle(-1);
connectionPool.setMaxWaitMillis(1000);
connectionPool.setValidationQuery(dbSettings.getValidationQuery());
connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = ON"));
if (foreignKeysEnabled) {
connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = ON"));
} else {
connectionPool.setConnectionInitSqls(Arrays.asList("PRAGMA foreign_keys = OFF"));
}
}
/**
* Lazily setup Singleton connection on first request.
*
* @param foreignKeys determines if foreign keys should be enforced during this connection for SQLite
*
* @return A connection from the connection pool.
*
* @throws EamDbException
*/
@Override
protected Connection connect() throws EamDbException {
protected Connection connect(boolean foreignKeys) throws EamDbException {
synchronized (this) {
if (!EamDb.isEnabled()) {
throw new EamDbException("Central Repository module is not enabled"); // NON-NLS
}
if (connectionPool == null) {
setupConnectionPool();
setupConnectionPool(foreignKeys);
}
try {
return connectionPool.getConnection();
} catch (SQLException ex) {
@ -198,6 +202,18 @@ final class SqliteEamDb extends AbstractSqlEamDb {
}
}
/**
* Lazily setup Singleton connection on first request with foreign keys enforced.
*
* @return A connection from the connection pool.
*
* @throws EamDbException
*/
@Override
protected Connection connect() throws EamDbException {
return connect(true);
}
@Override
protected String getConflictClause() {
// For sqlite, our conflict clause is part of the table schema
@ -275,7 +291,7 @@ final class SqliteEamDb extends AbstractSqlEamDb {
}
@Override
public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException{
public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException {
try {
acquireExclusiveLock();
super.addDataSourceObjectId(rowId, dataSourceObjectId);
@ -433,14 +449,14 @@ final class SqliteEamDb extends AbstractSqlEamDb {
releaseSharedLock();
}
}
/**
* Changes the name of a data source in the DB
*
* @param eamDataSource The data source
* @param newName The new name
*
* @throws EamDbException
*
* @param eamDataSource The data source
* @param newName The new name
*
* @throws EamDbException
*/
@Override
public void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException {
@ -451,7 +467,7 @@ final class SqliteEamDb extends AbstractSqlEamDb {
releaseExclusiveLock();
}
}
/**
* Updates the MD5 hash value in an existing data source in the database.
*
@ -466,7 +482,7 @@ final class SqliteEamDb extends AbstractSqlEamDb {
releaseExclusiveLock();
}
}
/**
* Updates the SHA-1 hash value in an existing data source in the database.
*
@ -481,9 +497,10 @@ final class SqliteEamDb extends AbstractSqlEamDb {
releaseExclusiveLock();
}
}
/**
* Updates the SHA-256 hash value in an existing data source in the database.
* Updates the SHA-256 hash value in an existing data source in the
* database.
*
* @param eamDataSource The data source to update
*/
@ -513,15 +530,6 @@ final class SqliteEamDb extends AbstractSqlEamDb {
}
}
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValue of the given eamArtifact.
*
* @param aType The type of the artifact
* @param value The correlation value
*
* @return List of artifact instances for a given type/value
*/
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException {
try {
@ -532,6 +540,26 @@ final class SqliteEamDb extends AbstractSqlEamDb {
}
}
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List<String> values) throws EamDbException, CorrelationAttributeNormalizationException {
try {
acquireSharedLock();
return super.getArtifactInstancesByTypeValues(aType, values);
} finally {
releaseSharedLock();
}
}
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List<String> values, List<Integer> caseIds) throws EamDbException, CorrelationAttributeNormalizationException {
try {
acquireSharedLock();
return super.getArtifactInstancesByTypeValuesAndCases(aType, values, caseIds);
} finally {
releaseSharedLock();
}
}
/**
* Retrieves eamArtifact instances from the database that are associated
* with the aType and filePath

109
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java
View File

@ -1,7 +1,7 @@
/*
* Central Repository
*
* Copyright 2015-2017 Basis Technology Corp.
* Copyright 2015-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -43,19 +43,19 @@ import static org.sleuthkit.autopsy.centralrepository.datamodel.AbstractSqlEamDb
public final class SqliteEamDbSettings {
private final static Logger LOGGER = Logger.getLogger(SqliteEamDbSettings.class.getName());
private final String DEFAULT_DBNAME = "central_repository.db"; // NON-NLS
private final String DEFAULT_DBDIRECTORY = PlatformUtil.getUserDirectory() + File.separator + "central_repository"; // NON-NLS
private final String JDBC_DRIVER = "org.sqlite.JDBC"; // NON-NLS
private final String JDBC_BASE_URI = "jdbc:sqlite:"; // NON-NLS
private final String VALIDATION_QUERY = "SELECT count(*) from sqlite_master"; // NON-NLS
private static final String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF";
private static final String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL";
private static final String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL";
private static final String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True";
private static final String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'";
private static final String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096";
private static final String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON";
private final String DB_NAMES_REGEX = "[a-z][a-z0-9_]*(\\.db)?";
private final static String DEFAULT_DBNAME = "central_repository.db"; // NON-NLS
private final static String DEFAULT_DBDIRECTORY = PlatformUtil.getUserDirectory() + File.separator + "central_repository"; // NON-NLS
private final static String JDBC_DRIVER = "org.sqlite.JDBC"; // NON-NLS
private final static String JDBC_BASE_URI = "jdbc:sqlite:"; // NON-NLS
private final static String VALIDATION_QUERY = "SELECT count(*) from sqlite_master"; // NON-NLS
private final static String PRAGMA_SYNC_OFF = "PRAGMA synchronous = OFF";
private final static String PRAGMA_SYNC_NORMAL = "PRAGMA synchronous = NORMAL";
private final static String PRAGMA_JOURNAL_WAL = "PRAGMA journal_mode = WAL";
private final static String PRAGMA_READ_UNCOMMITTED_TRUE = "PRAGMA read_uncommitted = True";
private final static String PRAGMA_ENCODING_UTF8 = "PRAGMA encoding = 'UTF-8'";
private final static String PRAGMA_PAGE_SIZE_4096 = "PRAGMA page_size = 4096";
private final static String PRAGMA_FOREIGN_KEYS_ON = "PRAGMA foreign_keys = ON";
private final static String DB_NAMES_REGEX = "[a-z][a-z0-9_]*(\\.db)?";
private String dbName;
private String dbDirectory;
private int bulkThreshold;
@ -282,23 +282,6 @@ public final class SqliteEamDbSettings {
String casesIdx1 = "CREATE INDEX IF NOT EXISTS cases_org_id ON cases (org_id)";
String casesIdx2 = "CREATE INDEX IF NOT EXISTS cases_case_uid ON cases (case_uid)";
StringBuilder createDataSourcesTable = new StringBuilder();
createDataSourcesTable.append("CREATE TABLE IF NOT EXISTS data_sources (");
createDataSourcesTable.append("id integer primary key autoincrement NOT NULL,");
createDataSourcesTable.append("case_id integer NOT NULL,");
createDataSourcesTable.append("device_id text NOT NULL,");
createDataSourcesTable.append("name text NOT NULL,");
createDataSourcesTable.append("datasource_obj_id integer,");
createDataSourcesTable.append("md5 text DEFAULT NULL,");
createDataSourcesTable.append("sha1 text DEFAULT NULL,");
createDataSourcesTable.append("sha256 text DEFAULT NULL,");
createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name)");
createDataSourcesTable.append(")");
String dataSourceIdx1 = "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)";
String dataSourceIdx2 = "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)";
StringBuilder createReferenceSetsTable = new StringBuilder();
createReferenceSetsTable.append("CREATE TABLE IF NOT EXISTS reference_sets (");
createReferenceSetsTable.append("id integer primary key autoincrement NOT NULL,");
@ -371,9 +354,9 @@ public final class SqliteEamDbSettings {
stmt.execute(casesIdx1);
stmt.execute(casesIdx2);
stmt.execute(createDataSourcesTable.toString());
stmt.execute(dataSourceIdx1);
stmt.execute(dataSourceIdx2);
stmt.execute(getCreateDataSourcesTableStatement());
stmt.execute(getAddDataSourcesNameIndexStatement());
stmt.execute(getAddDataSourcesObjectIdIndexStatement());
stmt.execute(createReferenceSetsTable.toString());
stmt.execute(referenceSetsIdx1);
@ -435,21 +418,49 @@ public final class SqliteEamDbSettings {
*/
static String getCreateArtifactInstancesTableTemplate() {
// Each "%s" will be replaced with the relevant TYPE_instances table name.
StringBuilder createArtifactInstancesTableTemplate = new StringBuilder();
createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s (");
createArtifactInstancesTableTemplate.append("id integer primary key autoincrement NOT NULL,");
createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,");
createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,");
createArtifactInstancesTableTemplate.append("value text NOT NULL,");
createArtifactInstancesTableTemplate.append("file_path text NOT NULL,");
createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,");
createArtifactInstancesTableTemplate.append("comment text,");
createArtifactInstancesTableTemplate.append("file_obj_id integer,");
createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,");
createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL");
createArtifactInstancesTableTemplate.append(")");
return createArtifactInstancesTableTemplate.toString();
return "CREATE TABLE IF NOT EXISTS %s (id integer primary key autoincrement NOT NULL,"
+ "case_id integer NOT NULL,data_source_id integer NOT NULL,value text NOT NULL,"
+ "file_path text NOT NULL,known_status integer NOT NULL,comment text,file_obj_id integer,"
+ "CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"
+ "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
+ "foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL)";
}
/**
* Get the statement String for creating a new data_sources table in a
* Sqlite central repository.
*
* @return a String which is a statement for cretating a new data_sources
* table
*/
static String getCreateDataSourcesTableStatement() {
return "CREATE TABLE IF NOT EXISTS data_sources (id integer primary key autoincrement NOT NULL,"
+ "case_id integer NOT NULL,device_id text NOT NULL,name text NOT NULL,datasource_obj_id integer,"
+ "md5 text DEFAULT NULL,sha1 text DEFAULT NULL,sha256 text DEFAULT NULL,"
+ "foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"
+ "CONSTRAINT datasource_unique UNIQUE (case_id, datasource_obj_id))";
}
/**
* Get the statement for creating an index on the name column of the
* data_sources table.
*
* @return a String which is a statement for adding an index on the name
* column of the data_sources table.
*/
static String getAddDataSourcesNameIndexStatement() {
return "CREATE INDEX IF NOT EXISTS data_sources_name ON data_sources (name)";
}
/**
* Get the statement for creating an index on the data_sources_object_id
* column of the data_sources table.
*
* @return a String which is a statement for adding an index on the
* data_sources_object_id column of the data_sources table.
*/
static String getAddDataSourcesObjectIdIndexStatement() {
return "CREATE INDEX IF NOT EXISTS data_sources_object_id ON data_sources (datasource_obj_id)";
}
/**

10
Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java
View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -54,17 +54,17 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
@Override
public CommonAttributeCountSearchResults findMatchesByCount() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase());
Set<String> mimeTypesToFilterOn = getMimeTypesToFilterOn();
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase(), mimeTypesToFilterOn);
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
}
@Override
public CommonAttributeCaseSearchResults findMatchesByCase() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCase(Case.getCurrentCase());
Set<String> mimeTypesToFilterOn = getMimeTypesToFilterOn();
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCase(Case.getCurrentCase(), mimeTypesToFilterOn);
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
}
@NbBundle.Messages({

51
Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java
View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -33,7 +33,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNor
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
/**
* Stores the results from the various types of common attribute searching
@ -55,11 +54,9 @@ final public class CommonAttributeCaseSearchResults {
* common, value of 0 is disabled
* @param resultType The type of Correlation Attribute being
* searched for
* @param mimeTypesToFilterOn Set of mime types to include for intercase
* searches
*/
CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId(), mimeTypesToFilterOn);
CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) {
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId());
}
/**
@ -71,7 +68,7 @@ final public class CommonAttributeCaseSearchResults {
* common, value of 0 is disabled
*/
CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold) {
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID, new HashSet<>());
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID);
}
/**
@ -110,11 +107,10 @@ final public class CommonAttributeCaseSearchResults {
* not be more common than
* @param resultTypeId the ID of the result type contained in the
* metadata
* @param mimeTypesToFilterOn the mimetypes to include in our results
*
* @return metadata
*/
private Map<String, Map<String, CommonAttributeValueList>> filterMetadata(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, int resultTypeId, Set<String> mimeTypesToFilterOn) {
private Map<String, Map<String, CommonAttributeValueList>> filterMetadata(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, int resultTypeId) {
try {
final String currentCaseName;
try {
@ -123,8 +119,9 @@ final public class CommonAttributeCaseSearchResults {
throw new EamDbException("Unable to get current case while performing filtering", ex);
}
Map<String, CommonAttributeValueList> currentCaseDataSourceMap = metadata.get(currentCaseName);
if (currentCaseDataSourceMap == null) {
throw new EamDbException("No data for current case found in results, indicating there are no results and nothing will be filtered");
Map<String, Map<String, CommonAttributeValueList>> filteredCaseNameToDataSourcesTree = new HashMap<>();
if (currentCaseDataSourceMap == null) { //there are no results
return filteredCaseNameToDataSourcesTree;
}
CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance
.getDefaultCorrelationTypes()
@ -133,13 +130,14 @@ final public class CommonAttributeCaseSearchResults {
.findFirst().get();
//Call countUniqueDataSources once to reduce the number of DB queries needed to get the frequencyPercentage
Double uniqueCaseDataSourceTuples = EamDb.getInstance().getCountUniqueDataSources().doubleValue();
Map<String, Map<String, CommonAttributeValueList>> filteredCaseNameToDataSourcesTree = new HashMap<>();
Map<String, CommonAttributeValue> valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn);
Map<String, CommonAttributeValue> valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples);
for (Entry<String, Map<String, CommonAttributeValueList>> mapOfDataSources : Collections.unmodifiableMap(metadata).entrySet()) {
if (!mapOfDataSources.getKey().equals(currentCaseName)) {
//rebuild the metadata structure with items from the current case substituted for their matches in other cases results we want to filter out removed
Map<String, CommonAttributeValueList> newTreeForCase = createTreeForCase(valuesToKeepCurrentCase, mapOfDataSources.getValue());
filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase);
if (!newTreeForCase.isEmpty()) {
filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase);
}
}
}
return filteredCaseNameToDataSourcesTree;
@ -162,21 +160,20 @@ final public class CommonAttributeCaseSearchResults {
* should not be more common than
* @param uniqueCaseDataSourceTuples the number of unique data sources in
* the CR
* @param mimeTypesToFilterOn the mimetypes to include in our results
*
* @return a map of correlation value to CommonAttributeValue for results
* from the current case
*
* @throws EamDbException
*/
private Map<String, CommonAttributeValue> getValuesToKeepFromCurrentCase(Map<String, CommonAttributeValueList> dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set<String> mimeTypesToFilterOn) throws EamDbException {
private Map<String, CommonAttributeValue> getValuesToKeepFromCurrentCase(Map<String, CommonAttributeValueList> dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples) throws EamDbException {
Map<String, CommonAttributeValue> valuesToKeep = new HashMap<>();
Set<String> valuesToRemove = new HashSet<>();
for (Entry<String, CommonAttributeValueList> mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataSet()) {
if (valuesToRemove.contains(value.getValue())) {
//do nothing this value will not be added
} else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn)) {
} else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples)) {
valuesToRemove.add(value.getValue());
} else {
valuesToKeep.put(value.getValue(), value);
@ -202,7 +199,7 @@ final public class CommonAttributeCaseSearchResults {
private Map<String, CommonAttributeValueList> createTreeForCase(Map<String, CommonAttributeValue> valuesToKeepCurrentCase, Map<String, CommonAttributeValueList> dataSourceToValueList) throws EamDbException {
Map<String, CommonAttributeValueList> treeForCase = new HashMap<>();
for (Entry<String, CommonAttributeValueList> mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataSet()) {
if (valuesToKeepCurrentCase.containsKey(value.getValue())) {
if (!treeForCase.containsKey(mapOfValueLists.getKey())) {
treeForCase.put(mapOfValueLists.getKey(), new CommonAttributeValueList());
@ -226,7 +223,6 @@ final public class CommonAttributeCaseSearchResults {
* should not be more common than
* @param uniqueCaseDataSourceTuples the number of unique data sources in
* the CR
* @param mimeTypesToInclude the mimetypes to include in our results
*
* @return true if the value should be filtered and removed from what is
* shown to the user, false if the value should not be removed and
@ -234,20 +230,7 @@ final public class CommonAttributeCaseSearchResults {
*
* @throws EamDbException
*/
private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set<String> mimeTypesToInclude) throws EamDbException {
//Intracase common attribute searches will have been created with an empty mimeTypesToInclude list
//because when performing intra case search this filtering will have been done during the query of the case database
if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
AbstractFile abstractFile = commonAttr.getAbstractFile();
if (abstractFile != null) {
String mimeType = abstractFile.getMIMEType();
if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
return true;
}
}
}
}
private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples) throws EamDbException {
if (maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set
try {
Double uniqueTypeValueTuples = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue(

49
Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java
View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -22,18 +22,15 @@ package org.sleuthkit.autopsy.commonpropertiessearch;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
/**
* Stores the results from the various types of common attribute searching
@ -45,7 +42,6 @@ final public class CommonAttributeCountSearchResults {
// maps instance count to list of attribute values.
private final Map<Integer, CommonAttributeValueList> instanceCountToAttributeValues;
private final Set<String> mimeTypesToInclude;
private final int percentageThreshold;
private final int resultTypeId;
@ -58,15 +54,13 @@ final public class CommonAttributeCountSearchResults {
* common, value of 0 is disabled
* @param resultType The type of Correlation Attribute being
* searched for
* @param mimeTypesToFilterOn Set of mime types to include for intercase
* searches
*
*/
CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) {
//wrap in a new object in case any client code has used an unmodifiable collection
this.instanceCountToAttributeValues = new HashMap<>(metadata);
this.percentageThreshold = percentageThreshold;
this.resultTypeId = resultType.getId();
this.mimeTypesToInclude = mimeTypesToFilterOn;
}
/**
@ -82,7 +76,6 @@ final public class CommonAttributeCountSearchResults {
this.instanceCountToAttributeValues = new HashMap<>(metadata);
this.percentageThreshold = percentageThreshold;
this.resultTypeId = CorrelationAttributeInstance.FILES_TYPE_ID;
this.mimeTypesToInclude = new HashSet<>(); //don't filter on mimetypes
}
/**
@ -152,36 +145,8 @@ final public class CommonAttributeCountSearchResults {
final Integer key = listOfValues.getKey();
final CommonAttributeValueList values = listOfValues.getValue();
for (CommonAttributeValue value : values.getDelayedMetadataList()) { // Need the real metadata
//Intracase common attribute searches will have been created with an empty mimeTypesToInclude list
//because when performing intra case search this filtering will have been done during the query of the case database
boolean mimeTypeToRemove = false; //allow code to be more efficient by not attempting to remove the same value multiple times
if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
AbstractFile abstractFile = commonAttr.getAbstractFile();
if (abstractFile != null) {
String mimeType = commonAttr.getAbstractFile().getMIMEType();
if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
if (itemsToRemove.containsKey(key)) {
itemsToRemove.get(key).add(value);
} else {
List<CommonAttributeValue> toRemove = new ArrayList<>();
toRemove.add(value);
itemsToRemove.put(key, toRemove);
}
//value will be removed as the mime type existed and was not in the set to be included
//because value is removed this value does not need to be checked further
mimeTypeToRemove = true;
break;
}
}
if (mimeTypeToRemove) {
break;
}
}
}
if (!mimeTypeToRemove && maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set
for (CommonAttributeValue value : values.getDelayedMetadataSet()) { // Need the real metadata
if (maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set
try {
Double uniqueTypeValueTuples = eamDb.getCountUniqueCaseDataSourceTuplesHavingTypeValue(
attributeType, value.getValue()).doubleValue();
@ -209,7 +174,7 @@ final public class CommonAttributeCountSearchResults {
final CommonAttributeValueList instanceCountValue = this.instanceCountToAttributeValues.get(key);
if (instanceCountValue != null) {
instanceCountValue.removeMetaData(value);
if (instanceCountValue.getDelayedMetadataList().isEmpty()) { // Check the real metadata
if (instanceCountValue.getDelayedMetadataSet().isEmpty()) { // Check the real metadata
this.instanceCountToAttributeValues.remove(key);
}
}
@ -226,7 +191,7 @@ final public class CommonAttributeCountSearchResults {
int count = 0;
for (CommonAttributeValueList data : this.instanceCountToAttributeValues.values()) {
for (CommonAttributeValue md5 : data.getDelayedMetadataList()) {
for (CommonAttributeValue md5 : data.getDelayedMetadataSet()) {
count += md5.getInstanceCount();
}
}

53
Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java
View File

@ -1,16 +1,16 @@
/*
*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
*
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -21,7 +21,9 @@ package org.sleuthkit.autopsy.commonpropertiessearch;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* Utility and wrapper model around data required for Common Files Search
@ -34,10 +36,10 @@ final public class CommonAttributeValueList {
* The list of value nodes, which begins empty.
*/
private final List<CommonAttributeValue> metadataList;
/**
* The backing list of value nodes, which will be dynamically loaded
* when requested.
* The backing list of value nodes, which will be dynamically loaded when
* requested.
*/
private final List<CommonAttributeValue> delayedMetadataList;
@ -58,36 +60,38 @@ final public class CommonAttributeValueList {
}
/**
* Get the list of value nodes. Will be empty if
* displayDelayedMetadata() has not been called for the
* parent InstanceCountNode
* Get the list of value nodes. Will be empty if displayDelayedMetadata()
* has not been called for the parent InstanceCountNode
*
* @return metadataList the list of nodes
*/
public List<CommonAttributeValue> getMetadataList() {
return Collections.unmodifiableList(this.metadataList);
}
/**
* Get the delayed list of value nodes. Only use for
* determining how many CommonAttributeValues
* actually exist in the list.
* @return metadataList the list of nodes
* Get the delayed set of value nodes. Only use for determining which values
* and how many CommonAttributeValues actually exist in the list.
*
* @return metadataList the set of nodes
*/
List<CommonAttributeValue> getDelayedMetadataList() {
return Collections.unmodifiableList(this.delayedMetadataList);
Set<CommonAttributeValue> getDelayedMetadataSet() {
//Allows nodes to be de-duped
return new HashSet<>(this.delayedMetadataList);
}
void removeMetaData(CommonAttributeValue commonVal) {
this.delayedMetadataList.remove(commonVal);
}
/**
* Return the size of the backing list, in case
* displayDelayedMetadata() has not be called yet.
* Return the size of the backing list, in case displayDelayedMetadata() has
* not be called yet.
*
* @return int the number of matches for this value
*/
int getCommonAttributeListSize() {
return this.delayedMetadataList.size();
return this.delayedMetadataList.size();
}
/**
@ -103,6 +107,7 @@ final public class CommonAttributeValueList {
/**
* A a value node to the list, to be loaded later.
*
* @param metadata the node to add
*/
void addMetadataToList(CommonAttributeValue metadata) {

7
Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java
View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -110,8 +110,7 @@ public final class InstanceDataSourceNode extends DisplayableItemNode {
}
/**
* ChildFactory which builds DisplayableItem from the metadata data
* sources.
* ChildFactory which builds DisplayableItem from the metadata data sources.
*/
static class FileInstanceNodeFactory extends ChildFactory<AbstractCommonAttributeInstance> {
@ -123,7 +122,7 @@ public final class InstanceDataSourceNode extends DisplayableItemNode {
@Override
protected boolean createKeys(List<AbstractCommonAttributeInstance> list) {
for (CommonAttributeValue value : descendants.getDelayedMetadataList()) {
for (CommonAttributeValue value : descendants.getDelayedMetadataSet()) {
// This is a bit of a hack to ensure that the AbstractFile instance
// has been created before createNodesForKey() is called. Constructing
// the AbstractFile in createNodesForKey() was resulting in UI lockups.

355
Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -18,10 +18,16 @@
*/
package org.sleuthkit.autopsy.commonpropertiessearch;
import com.google.common.collect.Iterables;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import org.sleuthkit.autopsy.casemodule.Case;
@ -32,12 +38,13 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil;
import org.sleuthkit.autopsy.centralrepository.datamodel.InstanceTableCallback;
import org.sleuthkit.autopsy.commonpropertiessearch.AbstractCommonAttributeInstance.NODE_TYPE;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.CaseDbAccessManager;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Used to process and return CorrelationCase values from the EamDB for
@ -45,23 +52,13 @@ import org.sleuthkit.datamodel.HashUtility;
*/
final class InterCaseSearchResultsProcessor {
private static final Logger LOGGER = Logger.getLogger(CommonAttributePanel.class.getName());
private static final String INTER_CASE_WHERE_CLAUSE = "case_id=%s AND (known_status !=%s OR known_status IS NULL)"; //NON-NLS
/**
* The CorrelationAttributeInstance.Type this Processor will query on
*/
private final Type correlationType;
private static final Logger LOGGER = Logger.getLogger(CommonAttributePanel.class.getName());
/**
* The initial CorrelationAttributeInstance ids lookup query.
*/
private final String interCaseWhereClause;
/**
* The single CorrelationAttributeInstance object retrieval query
*/
private final String singleInterCaseWhereClause;
/**
* Used in the InterCaseCommonAttributeSearchers to find common attribute
* instances and generate nodes at the UI level.
@ -71,32 +68,6 @@ final class InterCaseSearchResultsProcessor {
*/
InterCaseSearchResultsProcessor(CorrelationAttributeInstance.Type theType) {
this.correlationType = theType;
interCaseWhereClause = getInterCaseWhereClause();
singleInterCaseWhereClause = getSingleInterCaseWhereClause();
}
private String getInterCaseWhereClause() {
String tableName = EamDbUtil.correlationTypeToInstanceTableName(correlationType);
StringBuilder sqlString = new StringBuilder(250);
sqlString.append("value IN (SELECT value FROM ")
.append(tableName)
.append(" WHERE value IN (SELECT value FROM ")
.append(tableName)
.append(" WHERE case_id=%s AND (known_status !=%s OR known_status IS NULL) GROUP BY value)")
.append(" GROUP BY value HAVING COUNT(DISTINCT case_id) > 1) ORDER BY value");
return sqlString.toString();
}
private String getSingleInterCaseWhereClause() {
String tableName = EamDbUtil.correlationTypeToInstanceTableName(correlationType);
StringBuilder sqlString = new StringBuilder(250);
sqlString.append("value IN (SELECT value FROM ")
.append(tableName)
.append(" WHERE value IN (SELECT value FROM ")
.append(tableName)
.append(" WHERE case_id=%s AND (known_status !=%s OR known_status IS NULL) GROUP BY value)")
.append(" AND (case_id=%s OR case_id=%s) GROUP BY value HAVING COUNT(DISTINCT case_id) > 1) ORDER BY value");
return sqlString.toString();
}
/**
@ -122,30 +93,53 @@ final class InterCaseSearchResultsProcessor {
return null;
}
/**
* Get the portion of the select query which will get md5 values for files
* from the current case which are potentially being correlated on.
*
* @param mimeTypesToFilterOn the set of mime types to filter on
*
* @return the portion of a query which follows the SELECT keyword for
* finding MD5s which we are correlating on
*
* @throws EamDbException
*/
private String getFileQuery(Set<String> mimeTypesToFilterOn) throws EamDbException {
String query;
query = "md5 AS value FROM tsk_files WHERE known!=" + TskData.FileKnown.KNOWN.getFileKnownValue() + " AND md5 IS NOT NULL"; //NON-NLS
if (!mimeTypesToFilterOn.isEmpty()) {
query = query + " AND mime_type IS NOT NULL AND mime_type IN ('" + String.join("', '", mimeTypesToFilterOn) + "')"; //NON-NLS
}
return query;
}
/**
* Given the current case, fins all intercase common files from the EamDb
* and builds maps of case name to maps of data source name to
* CommonAttributeValueList.
*
* @param currentCase The current TSK Case.
* @param currentCase The current TSK Case.
* @param mimeTypesToFilterOn the set of mime types to filter on
*
* @return map of Case name to Maps of Datasources and their
* CommonAttributeValueLists
*/
Map<String, Map<String, CommonAttributeValueList>> findInterCaseValuesByCase(Case currentCase) {
Map<String, Map<String, CommonAttributeValueList>> findInterCaseValuesByCase(Case currentCase, Set<String> mimeTypesToFilterOn) {
try {
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId);
if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) {
currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback);
} else {
dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
}
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
} catch (EamDbException | TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
@ -153,24 +147,30 @@ final class InterCaseSearchResultsProcessor {
/**
* Given the current case, fins all intercase common files from the EamDb
* and builds maps of obj id to md5 and case.
* and builds maps of obj id to value and case.
*
* @param currentCase The current TSK Case.
* @param currentCase The current TSK Case.
* @param mimeTypesToFilterOn the set of mime types to filter on
*
* @return map of number of instances to CommonAttributeValueLists
*/
Map<Integer, CommonAttributeValueList> findInterCaseValuesByCount(Case currentCase) {
Map<Integer, CommonAttributeValueList> findInterCaseValuesByCount(Case currentCase, Set<String> mimeTypesToFilterOn) {
try {
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId);
if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) {
currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback);
} else {
dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
}
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
} catch (EamDbException | TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
@ -179,21 +179,30 @@ final class InterCaseSearchResultsProcessor {
/**
* Given the current case, and a specific case of interest, finds common
* files which exist between cases from the EamDb. Builds maps of obj id to
* md5 and case.
* value and case.
*
* @param currentCase The current TSK Case.
* @param singleCase The case of interest. Matches must exist in this case.
* @param currentCase The current TSK Case.
* @param mimeTypesToFilterOn the set of mime types to filter on
* @param singleCase The case of interest. Matches must exist in
* this case.
*
* @return map of number of instances to CommonAttributeValueLists
*/
Map<Integer, CommonAttributeValueList> findSingleInterCaseValuesByCount(Case currentCase, CorrelationCase singleCase) {
Map<Integer, CommonAttributeValueList> findSingleInterCaseValuesByCount(Case currentCase, Set<String> mimeTypesToFilterOn, CorrelationCase singleCase) {
try {
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
int targetCaseId = singleCase.getID();
dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback);
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId, targetCaseId);
if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) {
currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback);
} else {
dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
}
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
} catch (EamDbException | TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
@ -204,24 +213,30 @@ final class InterCaseSearchResultsProcessor {
* files which exist between cases from the EamDb. Builds map of case name
* to maps of data source name to CommonAttributeValueList.
*
* @param currentCase The current TSK Case.
* @param currentCase The current TSK Case.
* @param mimeTypesToFilterOn the set of mime types to filter on
* @param singleCase The case of interest. Matches must exist in
* this case.
*
* @return map of Case name to Maps of Datasources and their
* CommonAttributeValueLists
*
* @param currentCase The current TSK Case.
* @param singleCase The case of interest. Matches must exist in this case.
*/
Map<String, Map<String, CommonAttributeValueList>> findSingleInterCaseValuesByCase(Case currentCase, CorrelationCase singleCase) {
Map<String, Map<String, CommonAttributeValueList>> findSingleInterCaseValuesByCase(Case currentCase, Set<String> mimeTypesToFilterOn, CorrelationCase singleCase) {
try {
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
int targetCaseId = singleCase.getID();
dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback);
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId, targetCaseId);
if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) {
currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback);
} else {
dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
}
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
} catch (EamDbException | TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
@ -229,128 +244,146 @@ final class InterCaseSearchResultsProcessor {
/**
* Callback to use with findInterCaseValuesByCount which generates a list of
* md5s for common files search
* values for common property search
*/
private class InterCaseByCountCallback implements InstanceTableCallback {
private class InterCaseByCountCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback {
final Map<Integer, CommonAttributeValueList> instanceCollatedCommonFiles = new HashMap<>();
private final Map<Integer, CommonAttributeValueList> instanceCollatedCommonFiles = new HashMap<>();
private final int caseID;
private final int targetCase;
private CommonAttributeValue commonAttributeValue = null;
private String previousRowMd5 = "";
private InterCaseByCountCallback(int caseId) {
this(caseId, 0);
}
private InterCaseByCountCallback(int caseId, int targetCase) {
this.caseID = caseId;
this.targetCase = targetCase;
}
@Override
public void process(ResultSet resultSet) {
try {
Set<String> values = new HashSet<>();
List<Integer> targetCases = new ArrayList<>();
if (targetCase != 0) {
targetCases.add(caseID);
targetCases.add(targetCase);
}
while (resultSet.next()) {
int resultId = InstanceTableCallback.getId(resultSet);
String corValue = InstanceTableCallback.getValue(resultSet);
if (previousRowMd5.isEmpty()) {
previousRowMd5 = corValue;
}
if (corValue == null || HashUtility.isNoDataMd5(corValue)) {
continue;
}
countAndAddCommonAttributes(corValue, resultId);
values.add(corValue);
}
//Add the final instance(s)
if (commonAttributeValue != null) {
int size = commonAttributeValue.getInstanceCount();
if (instanceCollatedCommonFiles.containsKey(size)) {
instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue);
for (String corValue : values) {
List<CorrelationAttributeInstance> instances;
if (targetCases.isEmpty()) {
instances = EamDb.getInstance().getArtifactInstancesByTypeValues(correlationType, Arrays.asList(corValue));
} else {
CommonAttributeValueList value = new CommonAttributeValueList();
value.addMetadataToList(commonAttributeValue);
instanceCollatedCommonFiles.put(size, value);
instances = EamDb.getInstance().getArtifactInstancesByTypeValuesAndCases(correlationType, Arrays.asList(corValue), targetCases);
}
int size = instances.size();
if (size > 1) {
CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue);
boolean anotherCase = false;
for (CorrelationAttributeInstance instance : instances) {
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.COUNT_NODE);
searchResult.setCurrentAttributeInst(instance);
commonAttributeValue.addInstance(searchResult);
anotherCase = anotherCase || instance.getCorrelationCase().getID() != caseID;
}
if (anotherCase) {
if (instanceCollatedCommonFiles.containsKey(size)) {
instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue);
} else {
CommonAttributeValueList value = new CommonAttributeValueList();
value.addMetadataToList(commonAttributeValue);
instanceCollatedCommonFiles.put(size, value);
}
}
}
}
} catch (SQLException ex) {
} catch (SQLException | EamDbException | CorrelationAttributeNormalizationException ex) {
LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS
}
}
/**
* Add a resultId to the list of matches for a given corValue, which
* counts to number of instances of that match, determining which
* InstanceCountNode the match will be added to.
*
* @param corValue the value which matches
* @param resultId the CorrelationAttributeInstance id to be retrieved
* later.
*/
private void countAndAddCommonAttributes(String corValue, int resultId) {
if (commonAttributeValue == null) {
commonAttributeValue = new CommonAttributeValue(corValue);
}
if (!corValue.equals(previousRowMd5)) {
int size = commonAttributeValue.getInstanceCount();
if (instanceCollatedCommonFiles.containsKey(size)) {
instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue);
} else {
CommonAttributeValueList value = new CommonAttributeValueList();
value.addMetadataToList(commonAttributeValue);
instanceCollatedCommonFiles.put(size, value);
}
commonAttributeValue = new CommonAttributeValue(corValue);
previousRowMd5 = corValue;
}
// we don't *have* all the information for the rows in the CR,
// so we need to consult the present case via the SleuthkitCase object
// Later, when the FileInstanceNode is built. Therefore, build node generators for now.
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.COUNT_NODE);
CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId);
searchResult.setCurrentAttributeInst(corrAttr);
commonAttributeValue.addInstance(searchResult);
}
Map<Integer, CommonAttributeValueList> getInstanceCollatedCommonFiles() {
return Collections.unmodifiableMap(instanceCollatedCommonFiles);
}
}
/**
* Callback to use with findInterCaseValuesByCount which generates a list of
* md5s for common files search
* Callback to use with findInterCaseValuesByCase which generates a map of
* maps of values for common property search
*/
private class InterCaseByCaseCallback implements InstanceTableCallback {
private class InterCaseByCaseCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback {
final Map<String, Map<String, CommonAttributeValueList>> caseCollatedDataSourceCollections = new HashMap<>();
private static final int VALUE_BATCH_SIZE = 500;
private final Map<String, Map<String, CommonAttributeValueList>> caseCollatedDataSourceCollections = new HashMap<>();
private final int caseID;
private final int targetCase;
private InterCaseByCaseCallback(int caseId) {
this(caseId, 0);
}
private InterCaseByCaseCallback(int caseId, int targetCase) {
this.caseID = caseId;
this.targetCase = targetCase;
}
@Override
public void process(ResultSet resultSet) {
try {
List<Integer> targetCases = new ArrayList<>();
if (targetCase != 0) {
targetCases.add(caseID);
targetCases.add(targetCase);
}
Set<String> values = new HashSet<>();
while (resultSet.next()) {
int resultId = InstanceTableCallback.getId(resultSet);
String corValue = InstanceTableCallback.getValue(resultSet);
if (corValue == null || HashUtility.isNoDataMd5(corValue)) {
continue;
}
CorrelationCase correlationCase = EamDb.getInstance().getCaseById(InstanceTableCallback.getCaseId(resultSet));
String caseName = correlationCase.getDisplayName();
CorrelationDataSource correlationDatasource = EamDb.getInstance().getDataSourceById(correlationCase, InstanceTableCallback.getDataSourceId(resultSet));
//label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class
String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")";
if (!caseCollatedDataSourceCollections.containsKey(caseName)) {
caseCollatedDataSourceCollections.put(caseName, new HashMap<String, CommonAttributeValueList>());
}
Map<String, CommonAttributeValueList> dataSourceToFile = caseCollatedDataSourceCollections.get(caseName);
if (!dataSourceToFile.containsKey(dataSourceNameKey)) {
dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList());
}
CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey);
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.CASE_NODE);
CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId);
searchResult.setCurrentAttributeInst(corrAttr);
CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue);
commonAttributeValue.addInstance(searchResult);
valueList.addMetadataToList(commonAttributeValue);
dataSourceToFile.put(dataSourceNameKey, valueList);
caseCollatedDataSourceCollections.put(caseName, dataSourceToFile);
values.add(corValue);
}
} catch (EamDbException | SQLException ex) {
for (List<String> valuesChunk : Iterables.partition(values, VALUE_BATCH_SIZE)) {
List<CorrelationAttributeInstance> instances;
if (targetCases.isEmpty()) {
instances = EamDb.getInstance().getArtifactInstancesByTypeValues(correlationType, valuesChunk);
} else {
instances = EamDb.getInstance().getArtifactInstancesByTypeValuesAndCases(correlationType, valuesChunk, targetCases);
}
if (instances.size() > 1) {
for (CorrelationAttributeInstance instance : instances) {
CorrelationCase correlationCase = instance.getCorrelationCase();
String caseName = correlationCase.getDisplayName();
CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource();
//label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class
String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")";
if (!caseCollatedDataSourceCollections.containsKey(caseName)) {
caseCollatedDataSourceCollections.put(caseName, new HashMap<>());
}
Map<String, CommonAttributeValueList> dataSourceToFile = caseCollatedDataSourceCollections.get(caseName);
if (!dataSourceToFile.containsKey(dataSourceNameKey)) {
dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList());
}
CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey);
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE);
searchResult.setCurrentAttributeInst(instance);
CommonAttributeValue commonAttributeValue = new CommonAttributeValue(instance.getCorrelationValue());
commonAttributeValue.addInstance(searchResult);
valueList.addMetadataToList(commonAttributeValue);
dataSourceToFile.put(dataSourceNameKey, valueList);
caseCollatedDataSourceCollections.put(caseName, dataSourceToFile);
}
}
}
} catch (EamDbException | SQLException | CorrelationAttributeNormalizationException ex) {
LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS
}
}

11
Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java
View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -74,9 +74,10 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId);
this.correlationCaseName = correlationCase.getDisplayName();
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCount(Case.getCurrentCase(), correlationCase);
Set<String> mimeTypesToFilterOn = getMimeTypesToFilterOn();
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCount(Case.getCurrentCase(), mimeTypesToFilterOn, correlationCase);
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
}
/**
@ -96,10 +97,10 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId);
this.correlationCaseName = correlationCase.getDisplayName();
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCase(Case.getCurrentCase(), correlationCase);
Set<String> mimeTypesToFilterOn = getMimeTypesToFilterOn();
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCase(Case.getCurrentCase(), mimeTypesToFilterOn, correlationCase);
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
}
@NbBundle.Messages({

1
Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties
View File

@ -83,3 +83,4 @@ MediaViewImagePanel.zoomResetButton.text=Reset
MediaViewImagePanel.zoomTextField.text=
MediaViewImagePanel.rotationTextField.text=
MediaViewImagePanel.rotateLeftButton.toolTipText=
HtmlPanel.showImagesToggleButton.text=Show Images

6
Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED
View File

@ -32,6 +32,8 @@ GstVideoPanel.progress.buffering=Buffering...
GstVideoPanel.progressLabel.bufferingErr=Error buffering file
GstVideoPanel.progress.infoLabel.updateErr=Error updating video progress: {0}
GstVideoPanel.ExtractMedia.progress.buffering=Buffering {0}
HtmlPanel_showImagesToggleButton_hide=Hide Images
HtmlPanel_showImagesToggleButton_show=Show Images
MediaFileViewer.AccessibleContext.accessibleDescription=
MediaFileViewer.title=Media
MediaFileViewer.toolTip=Displays supported multimedia files (images, videos, audio)
@ -44,8 +46,6 @@ MediaViewVideoPanel.infoLabel.text=info
MediaViewImagePanel.imgFileTooLarge.msg=Could not load image file (too large): {0}
MessageContentViewer.AtrachmentsPanel.title=Attachments
MessageContentViewer.showImagesToggleButton.hide.text=Hide Images
MessageContentViewer.showImagesToggleButton.text=Show Images
MessageContentViewer.title=Message
MessageContentViewer.toolTip=Displays messages.
Metadata.nodeText.none=None
@ -53,6 +53,7 @@ Metadata.nodeText.truncated=(results truncated)
Metadata.nodeText.unknown=Unknown
Metadata.tableRowTitle.acquisitionDetails=Acquisition Details
Metadata.tableRowTitle.deviceId=Device ID
Metadata.tableRowTitle.downloadSource=Downloaded From
Metadata.tableRowTitle.imageType=Type
Metadata.tableRowTitle.mimeType=MIME Type
Metadata.tableRowTitle.name=Name
@ -139,6 +140,7 @@ MediaViewImagePanel.zoomResetButton.text=Reset
MediaViewImagePanel.zoomTextField.text=
MediaViewImagePanel.rotationTextField.text=
MediaViewImagePanel.rotateLeftButton.toolTipText=
HtmlPanel.showImagesToggleButton.text=Show Images
# {0} - tableName
SQLiteViewer.readTable.errorText=Error getting rows for table: {0}
# {0} - tableName

1
Core/src/org/sleuthkit/autopsy/contentviewers/FileViewer.form
View File

@ -11,6 +11,7 @@
<AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
<AuxValue name="designerSize" type="java.awt.Dimension" value="-84,-19,0,5,115,114,0,18,106,97,118,97,46,97,119,116,46,68,105,109,101,110,115,105,111,110,65,-114,-39,-41,-84,95,68,20,2,0,2,73,0,6,104,101,105,103,104,116,73,0,5,119,105,100,116,104,120,112,0,0,1,44,0,0,1,-112"/>
</AuxValues>
<LayoutCode>
<CodeStatement>

5
Core/src/org/sleuthkit/autopsy/contentviewers/FileViewer.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -50,7 +50,8 @@ public class FileViewer extends javax.swing.JPanel implements DataContentViewer
private final FileTypeViewer[] KNOWN_VIEWERS = new FileTypeViewer[]{
new SQLiteViewer(),
new PListViewer(),
new MediaFileViewer()
new MediaFileViewer(),
new HtmlViewer()
};
private FileTypeViewer lastViewer;

65
Core/src/org/sleuthkit/autopsy/contentviewers/HtmlPanel.form Executable file
View File

@ -0,0 +1,65 @@
<?xml version="1.0" encoding="UTF-8" ?>
<Form version="1.5" maxVersion="1.9" type="org.netbeans.modules.form.forminfo.JPanelFormInfo">
<AuxValues>
<AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_generateFQN" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_generateMnemonicsCode" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_i18nAutoMode" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_layoutCodeTarget" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
</AuxValues>
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="htmlScrollPane" pref="300" max="32767" attributes="0"/>
<Group type="102" attributes="0">
<Component id="showImagesToggleButton" min="-2" max="-2" attributes="0"/>
<EmptySpace min="0" pref="0" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<Component id="showImagesToggleButton" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="htmlScrollPane" pref="71" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
</Layout>
<SubComponents>
<Container class="javax.swing.JScrollPane" name="htmlScrollPane">
<Properties>
<Property name="verticalScrollBarPolicy" type="int" value="22"/>
</Properties>
<AuxValues>
<AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
</AuxValues>
<Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
<SubComponents>
<Component class="javax.swing.JTextPane" name="htmlbodyTextPane">
<Properties>
<Property name="editable" type="boolean" value="false"/>
</Properties>
</Component>
</SubComponents>
</Container>
<Component class="javax.swing.JToggleButton" name="showImagesToggleButton">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/contentviewers/Bundle.properties" key="HtmlPanel.showImagesToggleButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="showImagesToggleButtonActionPerformed"/>
</Events>
</Component>
</SubComponents>
</Form>

165
Core/src/org/sleuthkit/autopsy/contentviewers/HtmlPanel.java Executable file
View File

@ -0,0 +1,165 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.contentviewers;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.openide.util.NbBundle.Messages;
/**
* A file content viewer for HTML files.
*/
@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives
final class HtmlPanel extends javax.swing.JPanel {
private static final long serialVersionUID = 1L;
private String htmlText;
/**
* Creates new form HtmlViewerPanel
*/
HtmlPanel() {
initComponents();
Utilities.configureTextPaneAsHtml(htmlbodyTextPane);
}
/**
* Set the text pane's HTML text and refresh the view to display it.
*
* @param htmlText The HTML text to be applied to the text pane.
*/
void setHtmlText(String htmlText) {
this.htmlText = htmlText;
refresh();
}
/**
* Clear the HTML in the text pane and disable the show/hide button.
*/
void reset() {
htmlbodyTextPane.setText("");
showImagesToggleButton.setEnabled(false);
}
/**
* Guarantee the HTML text has 'html' and 'body' tags.
*
* @param htmlText The HTML text
*
* @return The HTML text with the 'html' and 'body' tags applied.
*/
private String wrapInHtmlBody(String htmlText) {
return "<html><body>" + htmlText + "</body></html>";
}
/**
* Cleans out input HTML string
*
* @param htmlInString The HTML string to cleanse
*
* @return The cleansed HTML String
*/
private String cleanseHTML(String htmlInString) {
Document doc = Jsoup.parse(htmlInString);
// Update all 'img' tags.
doc.select("img[src]").forEach(img -> img.attr("src", ""));
return doc.html();
}
/**
* Refresh the panel to reflect the current show/hide images setting.
*/
@Messages({
"HtmlPanel_showImagesToggleButton_show=Show Images",
"HtmlPanel_showImagesToggleButton_hide=Hide Images"
})
private void refresh() {
if (false == htmlText.isEmpty()) {
if (showImagesToggleButton.isSelected()) {
showImagesToggleButton.setText(Bundle.HtmlPanel_showImagesToggleButton_hide());
this.htmlbodyTextPane.setText(wrapInHtmlBody(htmlText));
} else {
showImagesToggleButton.setText(Bundle.HtmlPanel_showImagesToggleButton_show());
this.htmlbodyTextPane.setText(wrapInHtmlBody(cleanseHTML(htmlText)));
}
htmlbodyTextPane.setCaretPosition(0);
showImagesToggleButton.setEnabled(true);
}
}
/**
* This method is called from within the constructor to initialize the form.
* WARNING: Do NOT modify this code. The content of this method is always
* regenerated by the Form Editor.
*/
@SuppressWarnings("unchecked")
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
htmlScrollPane = new javax.swing.JScrollPane();
htmlbodyTextPane = new javax.swing.JTextPane();
showImagesToggleButton = new javax.swing.JToggleButton();
htmlScrollPane.setVerticalScrollBarPolicy(javax.swing.ScrollPaneConstants.VERTICAL_SCROLLBAR_ALWAYS);
htmlbodyTextPane.setEditable(false);
htmlScrollPane.setViewportView(htmlbodyTextPane);
org.openide.awt.Mnemonics.setLocalizedText(showImagesToggleButton, org.openide.util.NbBundle.getMessage(HtmlPanel.class, "HtmlPanel.showImagesToggleButton.text")); // NOI18N
showImagesToggleButton.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
showImagesToggleButtonActionPerformed(evt);
}
});
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(htmlScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 300, Short.MAX_VALUE)
.addGroup(layout.createSequentialGroup()
.addComponent(showImagesToggleButton)
.addGap(0, 0, Short.MAX_VALUE))
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addComponent(showImagesToggleButton)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(htmlScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 71, Short.MAX_VALUE))
);
}// </editor-fold>//GEN-END:initComponents
private void showImagesToggleButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_showImagesToggleButtonActionPerformed
refresh();
}//GEN-LAST:event_showImagesToggleButtonActionPerformed
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JScrollPane htmlScrollPane;
private javax.swing.JTextPane htmlbodyTextPane;
private javax.swing.JToggleButton showImagesToggleButton;
// End of variables declaration//GEN-END:variables
}

40
Core/src/org/sleuthkit/autopsy/contentviewers/HtmlViewer.form Executable file
View File

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8" ?>
<Form version="1.5" maxVersion="1.9" type="org.netbeans.modules.form.forminfo.JPanelFormInfo">
<AuxValues>
<AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_generateFQN" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_generateMnemonicsCode" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_i18nAutoMode" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_layoutCodeTarget" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
</AuxValues>
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="htmlPanel" max="32767" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="htmlPanel" max="32767" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
</Layout>
<SubComponents>
<Component class="org.sleuthkit.autopsy.contentviewers.HtmlPanel" name="htmlPanel">
</Component>
</SubComponents>
</Form>

128
Core/src/org/sleuthkit/autopsy/contentviewers/HtmlViewer.java Executable file
View File

@ -0,0 +1,128 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.contentviewers;
import java.awt.Component;
import java.awt.Cursor;
import java.util.Arrays;
import java.util.List;
import java.util.logging.Level;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException;
/**
* A file content viewer for HTML files.
*/
@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives
final class HtmlViewer extends javax.swing.JPanel implements FileTypeViewer {
private static final long serialVersionUID = 1L;
private static final Logger logger = Logger.getLogger(HtmlViewer.class.getName());
private static final String[] SUPPORTED_MIMETYPES = new String[]{
"text/html",
"application/xhtml+xml"
};
/**
* Creates new form HtmlViewerPanel
*/
HtmlViewer() {
initComponents();
}
/**
* Retrieve the HTML text content from the supplied file.
*
* @param abstractFile The file to read.
*
* @return The text content of the file.
*/
private String getHtmlText(AbstractFile abstractFile) {
try {
int fileSize = (int) abstractFile.getSize();
byte[] buffer = new byte[fileSize];
abstractFile.read(buffer, 0, fileSize);
return new String(buffer);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("Unable to read from file '%s' (id=%d).",
abstractFile.getName(), abstractFile.getId()), ex);
}
return null;
}
/**
* This method is called from within the constructor to initialize the form.
* WARNING: Do NOT modify this code. The content of this method is always
* regenerated by the Form Editor.
*/
@SuppressWarnings("unchecked")
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
htmlPanel = new org.sleuthkit.autopsy.contentviewers.HtmlPanel();
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(htmlPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addContainerGap())
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(htmlPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addContainerGap())
);
}// </editor-fold>//GEN-END:initComponents
// Variables declaration - do not modify//GEN-BEGIN:variables
private org.sleuthkit.autopsy.contentviewers.HtmlPanel htmlPanel;
// End of variables declaration//GEN-END:variables
@Override
public List<String> getSupportedMIMETypes() {
return Arrays.asList(SUPPORTED_MIMETYPES);
}
@Override
public void setFile(AbstractFile file) {
WindowManager.getDefault().getMainWindow().setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
htmlPanel.setHtmlText(getHtmlText(file));
WindowManager.getDefault().getMainWindow().setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
}
@Override
public Component getComponent() {
return this;
}
@Override
public void resetComponent() {
htmlPanel.reset();
}
}

5
Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java
View File

@ -126,7 +126,6 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan
scene.getStylesheets().add(MediaViewImagePanel.class.getResource("MediaViewImagePanel.css").toExternalForm()); //NOI18N
fxPanel.setScene(scene);
//bind size of image to that of scene, while keeping proportions
fxImageView.setSmooth(true);
fxImageView.setCache(true);
@ -142,11 +141,13 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan
}
/**
* clear the displayed image
* Clear the displayed image
*/
public void reset() {
Platform.runLater(() -> {
fxImageView.setViewport(new Rectangle2D(0, 0, 0, 0));
fxImageView.setImage(null);
scrollPane.setContent(null);
scrollPane.setContent(fxImageView);
});

39
Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.form
View File

@ -274,50 +274,17 @@
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="htmlScrollPane" alignment="0" max="32767" attributes="0"/>
<Group type="102" alignment="1" attributes="0">
<EmptySpace pref="533" max="32767" attributes="0"/>
<Component id="showImagesToggleButton" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="3" max="-2" attributes="0"/>
</Group>
<Component id="htmlPanel" alignment="0" pref="647" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<Component id="showImagesToggleButton" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="htmlScrollPane" pref="333" max="32767" attributes="0"/>
<EmptySpace min="-2" pref="0" max="-2" attributes="0"/>
</Group>
<Component id="htmlPanel" alignment="0" pref="362" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
</Layout>
<SubComponents>
<Container class="javax.swing.JScrollPane" name="htmlScrollPane">
<Properties>
<Property name="verticalScrollBarPolicy" type="int" value="22"/>
</Properties>
<AuxValues>
<AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
</AuxValues>
<Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
<SubComponents>
<Component class="javax.swing.JTextPane" name="htmlbodyTextPane">
<Properties>
<Property name="editable" type="boolean" value="false"/>
</Properties>
</Component>
</SubComponents>
</Container>
<Component class="javax.swing.JToggleButton" name="showImagesToggleButton">
<Properties>
<Property name="text" type="java.lang.String" value="Show Images"/>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="showImagesToggleButtonActionPerformed"/>
</Events>
<Component class="org.sleuthkit.autopsy.contentviewers.HtmlPanel" name="htmlPanel">
</Component>
</SubComponents>
</Container>

81
Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2017-2018 Basis Technology Corp.
* Copyright 2017-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -106,9 +106,12 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont
attachmentsScrollPane.setViewportView(drp);
msgbodyTabbedPane.setEnabledAt(ATTM_TAB_INDEX, true);
textAreas = Arrays.asList(headersTextArea, textbodyTextArea, htmlbodyTextPane, rtfbodyTextPane);
/*
* HTML tab uses the HtmlPanel instead of an internal text pane, so we
* use 'null' for that index.
*/
textAreas = Arrays.asList(headersTextArea, textbodyTextArea, null, rtfbodyTextPane);
Utilities.configureTextPaneAsHtml(htmlbodyTextPane);
Utilities.configureTextPaneAsRtf(rtfbodyTextPane);
resetComponent();
@ -150,9 +153,7 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont
textbodyScrollPane = new javax.swing.JScrollPane();
textbodyTextArea = new javax.swing.JTextArea();
htmlPane = new javax.swing.JPanel();
htmlScrollPane = new javax.swing.JScrollPane();
htmlbodyTextPane = new javax.swing.JTextPane();
showImagesToggleButton = new javax.swing.JToggleButton();
htmlPanel = new org.sleuthkit.autopsy.contentviewers.HtmlPanel();
rtfbodyScrollPane = new javax.swing.JScrollPane();
rtfbodyTextPane = new javax.swing.JTextPane();
attachmentsPanel = new javax.swing.JPanel();
@ -265,35 +266,15 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont
msgbodyTabbedPane.addTab(org.openide.util.NbBundle.getMessage(MessageContentViewer.class, "MessageContentViewer.textbodyScrollPane.TabConstraints.tabTitle"), textbodyScrollPane); // NOI18N
htmlScrollPane.setVerticalScrollBarPolicy(javax.swing.ScrollPaneConstants.VERTICAL_SCROLLBAR_ALWAYS);
htmlbodyTextPane.setEditable(false);
htmlScrollPane.setViewportView(htmlbodyTextPane);
org.openide.awt.Mnemonics.setLocalizedText(showImagesToggleButton, "Show Images");
showImagesToggleButton.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
showImagesToggleButtonActionPerformed(evt);
}
});
javax.swing.GroupLayout htmlPaneLayout = new javax.swing.GroupLayout(htmlPane);
htmlPane.setLayout(htmlPaneLayout);
htmlPaneLayout.setHorizontalGroup(
htmlPaneLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(htmlScrollPane)
.addGroup(javax.swing.GroupLayout.Alignment.TRAILING, htmlPaneLayout.createSequentialGroup()
.addContainerGap(533, Short.MAX_VALUE)
.addComponent(showImagesToggleButton)
.addGap(3, 3, 3))
.addComponent(htmlPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 647, Short.MAX_VALUE)
);
htmlPaneLayout.setVerticalGroup(
htmlPaneLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(htmlPaneLayout.createSequentialGroup()
.addComponent(showImagesToggleButton)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(htmlScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 333, Short.MAX_VALUE)
.addGap(0, 0, 0))
.addComponent(htmlPanel, javax.swing.GroupLayout.DEFAULT_SIZE, 362, Short.MAX_VALUE)
);
msgbodyTabbedPane.addTab(org.openide.util.NbBundle.getMessage(MessageContentViewer.class, "MessageContentViewer.htmlPane.TabConstraints.tabTitle"), htmlPane); // NOI18N
@ -358,26 +339,6 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont
);
}// </editor-fold>//GEN-END:initComponents
@NbBundle.Messages({
"MessageContentViewer.showImagesToggleButton.hide.text=Hide Images",
"MessageContentViewer.showImagesToggleButton.text=Show Images"})
private void showImagesToggleButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_showImagesToggleButtonActionPerformed
try {
String htmlText = getAttributeValueSafe(artifact, TSK_EMAIL_CONTENT_HTML);
if (false == htmlText.isEmpty()) {
if (showImagesToggleButton.isSelected()) {
showImagesToggleButton.setText(Bundle.MessageContentViewer_showImagesToggleButton_hide_text());
this.htmlbodyTextPane.setText(wrapInHtmlBody(htmlText));
} else {
showImagesToggleButton.setText(Bundle.MessageContentViewer_showImagesToggleButton_text());
this.htmlbodyTextPane.setText(wrapInHtmlBody(cleanseHTML(htmlText)));
}
}
} catch (TskCoreException ex) {
LOGGER.log(Level.WARNING, "Failed to get attributes for email message.", ex); //NON-NLS
}
}//GEN-LAST:event_showImagesToggleButtonActionPerformed
private void viewInNewWindowButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_viewInNewWindowButtonActionPerformed
new NewWindowViewAction("View in new window", drpExplorerManager.getSelectedNodes()[0]).actionPerformed(evt);
}//GEN-LAST:event_viewInNewWindowButtonActionPerformed
@ -396,12 +357,10 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont
private javax.swing.JScrollPane headersScrollPane;
private javax.swing.JTextArea headersTextArea;
private javax.swing.JPanel htmlPane;
private javax.swing.JScrollPane htmlScrollPane;
private javax.swing.JTextPane htmlbodyTextPane;
private org.sleuthkit.autopsy.contentviewers.HtmlPanel htmlPanel;
private javax.swing.JTabbedPane msgbodyTabbedPane;
private javax.swing.JScrollPane rtfbodyScrollPane;
private javax.swing.JTextPane rtfbodyTextPane;
private javax.swing.JToggleButton showImagesToggleButton;
private javax.swing.JLabel subjectLabel;
private javax.swing.JLabel subjectText;
private javax.swing.JScrollPane textbodyScrollPane;
@ -505,9 +464,8 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont
headersTextArea.setText("");
rtfbodyTextPane.setText("");
htmlbodyTextPane.setText("");
htmlPanel.reset();
textbodyTextArea.setText("");
showImagesToggleButton.setEnabled(false);
msgbodyTabbedPane.setEnabled(false);
}
@ -567,12 +525,15 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont
String attributeText = getAttributeValueSafe(artifact, type);
if (index == HTML_TAB_INDEX && StringUtils.isNotBlank(attributeText)) {
//special case for HTML, we need to 'cleanse' it
attributeText = wrapInHtmlBody(cleanseHTML(attributeText));
htmlPanel.setHtmlText(attributeText);
} else {
JTextComponent textComponent = textAreas.get(index);
if (textComponent != null) {
textComponent.setText(attributeText);
textComponent.setCaretPosition(0); //make sure we start at the top
}
}
JTextComponent textComponent = textAreas.get(index);
textComponent.setText(attributeText);
textComponent.setCaretPosition(0); //make sure we start at the top
final boolean hasText = attributeText.length() > 0;
msgbodyTabbedPane.setEnabledAt(index, hasText);
@ -613,10 +574,6 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont
directionText.setEnabled(false);
ccLabel.setEnabled(true);
showImagesToggleButton.setEnabled(true);
showImagesToggleButton.setText("Show Images");
showImagesToggleButton.setSelected(false);
try {
this.fromText.setText(getAttributeValueSafe(artifact, TSK_EMAIL_FROM));
this.fromText.setToolTipText(getAttributeValueSafe(artifact, TSK_EMAIL_FROM));

19
Core/src/org/sleuthkit/autopsy/contentviewers/Metadata.java
View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.contentviewers;
import java.awt.Component;
import java.util.List;
import java.util.logging.Level;
import org.apache.commons.lang3.StringUtils;
import org.openide.nodes.Node;
@ -29,6 +30,10 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataContentViewer;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.Image;
import org.sleuthkit.datamodel.FsContent;
@ -138,6 +143,7 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer {
"Metadata.tableRowTitle.timezone=Time Zone",
"Metadata.tableRowTitle.deviceId=Device ID",
"Metadata.tableRowTitle.acquisitionDetails=Acquisition Details",
"Metadata.tableRowTitle.downloadSource=Downloaded From",
"Metadata.nodeText.unknown=Unknown",
"Metadata.nodeText.none=None"})
@Override
@ -184,6 +190,19 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer {
addRow(sb, NbBundle.getMessage(this.getClass(), "Metadata.tableRowTitle.localPath"), file.getLocalAbsPath());
}
try {
List<BlackboardArtifact> sourceArtifacts = file.getArtifacts(ARTIFACT_TYPE.TSK_DOWNLOAD_SOURCE);
if (!sourceArtifacts.isEmpty()) {
BlackboardArtifact artifact = sourceArtifacts.get(0);
BlackboardAttribute urlAttr = artifact.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_URL));
if (urlAttr != null) {
addRow(sb, NbBundle.getMessage(this.getClass(), "Metadata.tableRowTitle.downloadSource"), urlAttr.getValueString());
}
}
} catch (TskCoreException ex) {
sb.append(NbBundle.getMessage(this.getClass(), "Metadata.nodeText.exceptionNotice.text")).append(ex.getLocalizedMessage());
}
endTable(sb);
/*

2
Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java
View File

@ -51,6 +51,7 @@ import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTER
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_TL_EVENT;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_DOWNLOAD_SOURCE;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskException;
@ -238,6 +239,7 @@ public class ExtractedContent implements AutopsyVisitableItem {
doNotShow.add(new BlackboardArtifact.Type(TSK_ACCOUNT));
doNotShow.add(new BlackboardArtifact.Type(TSK_TL_EVENT));
doNotShow.add(new BlackboardArtifact.Type(TSK_DATA_SOURCE_USAGE));
doNotShow.add(new BlackboardArtifact.Type(TSK_DOWNLOAD_SOURCE) );
}
private final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> {

4
Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/MSOfficeEmbeddedContentExtractor.java
View File

@ -302,8 +302,9 @@ class MSOfficeEmbeddedContentExtractor {
}
List<ExtractedFile> listOfExtractedImages = new ArrayList<>();
byte[] data = null;
int pictureNumber = 0; //added to ensure uniqueness in cases where suggestFullFileName returns duplicates
for (Picture picture : listOfAllPictures) {
String fileName = picture.suggestFullFileName();
String fileName = UNKNOWN_IMAGE_NAME_PREFIX +pictureNumber +"."+ picture.suggestFileExtension();
try {
data = picture.getContent();
} catch (Exception ex) {
@ -312,6 +313,7 @@ class MSOfficeEmbeddedContentExtractor {
writeExtractedImage(Paths.get(outputFolderPath, fileName).toString(), data);
// TODO Extract more info from the Picture viz ctime, crtime, atime, mtime
listOfExtractedImages.add(new ExtractedFile(fileName, getFileRelativePath(fileName), picture.getSize()));
pictureNumber++;
}
return listOfExtractedImages;

2
CoreLibs/ivy.xml
View File

@ -69,7 +69,7 @@
<dependency conf="autopsy_core->default" org="com.twelvemonkeys.imageio" name="imageio-thumbsdb" rev="3.2" />
<dependency conf="autopsy_core->default" org="com.twelvemonkeys.imageio" name="imageio-core" rev="3.2" />
<dependency conf="autopsy_core->default" org="com.twelvemonkeys.imageio" name="imageio-metadata" rev="3.2" />
<dependency conf="autopsy_core->default" org="com.googlecode.plist" name="dd-plist" rev="1.20"/>
<!-- conflict resolutions for multiple JAR versions -->
<conflict org="net.java.dev.jna" module="jna" rev="3.4.0"/>
<conflict org="net.java.dev.jna" module="platform" rev="3.4.0"/>

1
CoreLibs/nbproject/project.properties
View File

@ -75,6 +75,7 @@ file.reference.slf4j-simple-1.6.1.jar=release/modules/ext/slf4j-simple-1.6.1.jar
file.reference.stax-api-1.0.1.jar=release/modules/ext/stax-api-1.0.1.jar
file.reference.xml-apis-1.0.b2.jar=release/modules/ext/xml-apis-1.0.b2.jar
file.reference.xmlbeans-2.6.0.jar=release/modules/ext/xmlbeans-2.6.0.jar
file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar
javac.source=1.8
javac.compilerargs=-Xlint -Xlint:-serial
javadoc.reference.commons-csv-1.4.jar=release/modules/ext/commons-csv-1.4-javadoc.jar

21
CoreLibs/nbproject/project.xml
View File

@ -37,6 +37,7 @@
<package>com.apple.eawt</package>
<package>com.apple.eawt.event</package>
<package>com.apple.eio</package>
<package>com.dd.plist</package>
<package>com.github.lgooddatepicker.components</package>
<package>com.github.lgooddatepicker.optionalusertools</package>
<package>com.github.lgooddatepicker.zinternaltools</package>
@ -742,18 +743,10 @@
<runtime-relative-path>ext/jna-3.4.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jna-3.4.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/gson-2.8.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/gson-2.8.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jfxtras-common-8.0-r4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jfxtras-common-8.0-r4.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/opencv-248.jar</runtime-relative-path>
<binary-origin>release/modules/ext/opencv-248.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jsr305-1.3.9.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jsr305-1.3.9.jar</binary-origin>
@ -874,6 +867,14 @@
<runtime-relative-path>ext/commons-codec-1.10.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-codec-1.10.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/gson-2.8.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/gson-2.8.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/opencv-248.jar</runtime-relative-path>
<binary-origin>release/modules/ext/opencv-248.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/slf4j-simple-1.6.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/slf4j-simple-1.6.1.jar</binary-origin>
@ -970,6 +971,10 @@
<runtime-relative-path>ext/gst1-java-core-0.9.3.jar</runtime-relative-path>
<binary-origin>release/modules/ext/gst1-java-core-0.9.3.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/dd-plist-1.20.jar</runtime-relative-path>
<binary-origin>release/modules/ext/dd-plist-1.20.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/dom4j-1.6.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/dom4j-1.6.1.jar</binary-origin>

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
View File

@ -1,5 +1,5 @@
OpenIDE-Module-Display-Category=Ingest Module
OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\n\The module indexes files found in the disk image at ingest time.\n\It then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword seach bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found.
OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found.
OpenIDE-Module-Name=KeywordSearch
OptionsCategory_Name_KeywordSearchOptions=Keyword Search
OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search

3
RecentActivity/build.xml
View File

@ -19,6 +19,9 @@
<copy todir="${basedir}/release/rr-full" >
<fileset dir="${thirdparty.dir}/rr-full/" />
</copy>
<copy todir="${basedir}/release/ESEDatabaseView" >
<fileset dir="${thirdparty.dir}/ESEDatabaseView/" />
</copy>
</target>

434
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/BinaryCookieReader.java Executable file
View File

@ -0,0 +1,434 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.recentactivity;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.recentactivity.BinaryCookieReader.Cookie;
/**
* The binary cookie reader encapsulates all the knowledge of how to read the mac
* .binarycookie files into one class.
*
* The binarycookie file has a header which describes how many pages of cookies
* and where they are located. Each cookie page has a header and a list of
* cookies.
*
*/
public final class BinaryCookieReader implements Iterable<Cookie> {
private static final int MAGIC_SIZE = 4;
private static final int SIZEOF_INT_BYTES = 4;
private static final int PAGE_HEADER_VALUE = 256;
private static final String COOKIE_MAGIC = "cook"; //NON-NLS
private static final int MAC_EPOC_FIX = 978307200;
private final int[] pageSizeArray;
private final File cookieFile;
private static final Logger LOG = Logger.getLogger(BinaryCookieReader.class.getName());
/**
* The binary cookie reader encapsulates all the knowledge of how to read the mac
* .binarycookie files into one class.
*
*/
private BinaryCookieReader(File cookieFile, int[] pageSizeArray) {
this.cookieFile = cookieFile;
this.pageSizeArray = pageSizeArray;
}
/**
* initalizeReader opens the given file, reads the header and checks that
* the file is a binarycookie file. This function does not keep the file
* open.
*
* @param file binarycookie file
* @return An instance of the reader
* @throws FileNotFoundException
* @throws IOException
*/
public static BinaryCookieReader initalizeReader(File cookieFile) throws FileNotFoundException, IOException {
BinaryCookieReader reader = null;
try (DataInputStream dataStream = new DataInputStream(new FileInputStream(cookieFile))) {
byte[] magic = new byte[MAGIC_SIZE];
if (dataStream.read(magic) != MAGIC_SIZE) {
throw new IOException("Failed to read header, invalid file size (" + cookieFile.getName() + ")"); //NON-NLS
}
if (!(new String(magic)).equals(COOKIE_MAGIC)) {
throw new IOException(cookieFile.getName() + " is not a cookie file"); //NON-NLS
}
int[] sizeArray = null;
int pageCount = dataStream.readInt();
if (pageCount != 0) {
sizeArray = new int[pageCount];
for (int cnt = 0; cnt < pageCount; cnt++) {
sizeArray[cnt] = dataStream.readInt();
}
LOG.log(Level.INFO, "No cookies found in {0}", cookieFile.getName()); //NON-NLS
}
reader = new BinaryCookieReader(cookieFile, sizeArray);
}
return reader;
}
/**
* Creates and returns a instance of CookiePageIterator.
*
* @return CookiePageIterator
*/
@Override
public Iterator<Cookie> iterator() {
return new CookiePageIterator();
}
/**
* The cookiePageIterator iterates the binarycookie file by page.
*/
private class CookiePageIterator implements Iterator<Cookie> {
int pageIndex = 0;
CookiePage currentPage = null;
Iterator<Cookie> currentIterator = null;
DataInputStream dataStream = null;
/**
* The cookiePageIterator iterates the binarycookie file by page.
*/
CookiePageIterator() {
if(pageSizeArray == null || pageSizeArray.length == 0) {
return;
}
try {
dataStream = new DataInputStream(new FileInputStream(cookieFile));
// skip to the first page
dataStream.skipBytes((2 * SIZEOF_INT_BYTES) + (pageSizeArray.length * SIZEOF_INT_BYTES));
} catch (IOException ex) {
String errorMessage = String.format("An error occurred creating an input stream for %s", cookieFile.getName());
LOG.log(Level.WARNING, errorMessage, ex); //NON-NLS
closeStream(); // Just incase the error was from skip
}
}
/**
* Returns true if there are more cookies in the binarycookie file.
*
* @return True if there are more cookies
*/
@Override
public boolean hasNext() {
if (dataStream == null) {
return false;
}
if (currentIterator == null || !currentIterator.hasNext()) {
try {
if (pageIndex < pageSizeArray.length) {
byte[] nextPage = new byte[pageSizeArray[pageIndex]];
dataStream.read(nextPage);
currentPage = new CookiePage(nextPage);
currentIterator = currentPage.iterator();
} else {
closeStream();
return false;
}
pageIndex++;
} catch (IOException ex) {
closeStream();
String errorMessage = String.format("A read error occured for file %s (pageIndex = %d)", cookieFile.getName(), pageIndex);
LOG.log(Level.WARNING, errorMessage, ex); //NON-NLS
return false;
}
}
return currentIterator.hasNext();
}
/**
* Get the next cookie from the current CookieIterator.
*
* @return The next cookie
*/
@Override
public Cookie next() {
// Just in case someone uses next without hasNext, this check will
// make sure there are more elements and that we iterate properly
// through the pages.
if (!hasNext()) {
throw new NoSuchElementException();
}
return currentIterator.next();
}
/**
* Close the DataInputStream
*/
private void closeStream() {
if (dataStream != null) {
try {
dataStream.close();
dataStream = null;
} catch (IOException ex) {
String errorMessage = String.format("An error occurred trying to close stream for file %s", cookieFile.getName());
LOG.log(Level.WARNING, errorMessage, ex); //NON-NLS
}
}
}
}
/**
* Wrapper class for an instance of a CookiePage in the binarycookie file.
*/
private class CookiePage implements Iterable<Cookie> {
int[] cookieOffSets;
ByteBuffer pageBuffer;
/**
* Setup the CookiePage object. Calidates that the page bytes are in the
* correct format by checking for the header value of 0x0100.
*
* @param page byte array representing a cookie page
* @throws IOException
*/
CookiePage(byte[] page) throws IOException {
if (page == null || page.length == 0) {
throw new IllegalArgumentException("Invalid value for page passed to CookiePage constructor"); //NON-NLS
}
pageBuffer = ByteBuffer.wrap(page);
if (pageBuffer.getInt() != PAGE_HEADER_VALUE) {
pageBuffer = null;
throw new IOException("Invalid file format, bad page head value found"); //NON-NLS
}
pageBuffer.order(ByteOrder.LITTLE_ENDIAN);
int count = pageBuffer.getInt();
cookieOffSets = new int[count];
for (int cnt = 0; cnt < count; cnt++) {
cookieOffSets[cnt] = pageBuffer.getInt();
}
pageBuffer.getInt(); // All 0, not needed
}
/**
* Returns an instance of a CookieIterator.
*
* @return CookieIterator
*/
@Override
public Iterator<Cookie> iterator() {
return new CookieIterator();
}
/**
* Implements Iterator to iterate over the cookies in the page.
*/
private class CookieIterator implements Iterator<Cookie> {
int index = 0;
/**
* Checks to see if there are more cookies.
*
* @return True if there are more cookies, false if there are not
*/
@Override
public boolean hasNext() {
if (pageBuffer == null) {
return false;
}
return index < cookieOffSets.length;
}
/**
* Gets the next cookie from the page.
*
* @return Next cookie
*/
@Override
public Cookie next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
int offset = cookieOffSets[index];
int size = pageBuffer.getInt(offset);
byte[] cookieBytes = new byte[size];
pageBuffer.get(cookieBytes, 0, size);
index++;
return new Cookie(cookieBytes);
}
}
}
/**
* Represents an instance of a cookie from the binarycookie file.
*/
public class Cookie {
private final static int COOKIE_HEAD_SKIP = 16;
private final double expirationDate;
private final double creationDate;
private final String name;
private final String url;
private final String path;
private final String value;
/**
* Creates a cookie object from the given array of bytes.
*
* @param cookieBytes Byte array for the cookie
*/
protected Cookie(byte[] cookieBytes) {
if (cookieBytes == null || cookieBytes.length == 0) {
throw new IllegalArgumentException("Invalid value for cookieBytes passed to Cookie constructor"); //NON-NLS
}
ByteBuffer byteBuffer = ByteBuffer.wrap(cookieBytes);
byteBuffer.order(ByteOrder.LITTLE_ENDIAN);
// Skip past the four int values that we are not interested in
byteBuffer.position(byteBuffer.position() + COOKIE_HEAD_SKIP);
int urlOffset = byteBuffer.getInt();
int nameOffset = byteBuffer.getInt();
int pathOffset = byteBuffer.getInt();
int valueOffset = byteBuffer.getInt();
byteBuffer.getLong(); // 8 bytes of not needed
expirationDate = byteBuffer.getDouble();
creationDate = byteBuffer.getDouble();
url = decodeString(cookieBytes, urlOffset);
name = decodeString(cookieBytes, nameOffset);
path = decodeString(cookieBytes, pathOffset);
value = decodeString(cookieBytes, valueOffset);
}
/**
* Returns the expiration date of the cookie represented by this cookie
* object.
*
* @return Cookie expiration date in milliseconds with java epoch
*/
public final Long getExpirationDate() {
return ((long)expirationDate) + MAC_EPOC_FIX;
}
/**
* Returns the creation date of the cookie represented by this cookie
* object.
*
* @return Cookie creation date in milliseconds with java epoch
*/
public final Long getCreationDate() {
return ((long)creationDate) + MAC_EPOC_FIX;
}
/**
* Returns the url of the cookie represented by this cookie object.
*
* @return the cookie URL
*/
public final String getURL() {
return url;
}
/**
* Returns the name of the cookie represented by this cookie object.
*
* @return The cookie name
*/
public final String getName() {
return name;
}
/**
* Returns the path of the cookie represented by this cookie object.
*
* @return The cookie path
*/
public final String getPath() {
return path;
}
/**
* Returns the value of the cookie represented by this cookie object.
*
* @return The cookie value
*/
public final String getValue() {
return value;
}
/**
* Creates an ascii string from the bytes in byteArray starting at
* offset ending at the first null terminator found.
*
* @param byteArray Array of bytes
* @param offset starting offset in the array
* @return String with bytes converted to ascii
*/
private String decodeString(byte[] byteArray, int offset) {
byte[] stringBytes = new byte[byteArray.length - offset];
for (int index = 0; index < stringBytes.length; index++) {
byte nibble = byteArray[offset + index];
if (nibble != '\0') { //NON-NLS
stringBytes[index] = nibble;
} else {
break;
}
}
return new String(stringBytes);
}
}
}

3
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties
View File

@ -22,8 +22,7 @@ Chrome.getAutofill.errMsg.errGettingFiles=Error when trying to get Chrome Web Da
Chrome.getAutofill.errMsg.errAnalyzingFiles={0}: Error while trying to analyze file:{1}
Extract.dbConn.errMsg.failedToQueryDb={0}: Failed to query database.
ExtractIE.moduleName.text=Internet Explorer
ExtractIE.getBookmark.errMsg.errGettingBookmarks={0}: Error getting Internet Explorer Bookmarks.
ExtractIE.parentModuleName.noSpace=RecentActivity
ExtractIE.getBookmark.ere.noSpace=RecentActivity
ExtractIE.parentModuleName=Recent Activity
ExtractIE.getURLFromIEBmkFile.errMsg={0}: Error parsing IE bookmark File {1}
ExtractIE.getURLFromIEBmkFile.errMsg2={0}: Error parsing IE bookmark File {1}

56
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED
View File

@ -1,12 +1,24 @@
cannotBuildXmlParser=Unable to build XML parser:
cannotLoadSEUQA=Unable to load Search Engine URL Query Analyzer settings file, SEUQAMappings.xml:
cannotParseXml=Unable to parse XML file:
# {0} - OS name
ChromeCacheExtractor.moduleName=ChromeCacheExtractor
DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})
DataSourceUsageAnalyzer.parentModuleName=Recent Activity
Extract.indexError.message=Failed to index artifact for keyword search.
Extract.noOpenCase.errMsg=No open case available.
#{0} - the module name
ExtractEdge_getHistory_containerFileNotFound=Error while trying to analyze Edge history
ExtractEdge_Module_Name=Microsoft Edge
ExtractEdge_process_errMsg_errGettingWebCacheFiles=Error trying to retrieving Edge WebCacheV01 file
ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file
ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer
ExtractEdge_process_errMsg_webcacheFail=Failure processing Microsoft Edge WebCacheV01.dat file
ExtractIE.getBookmark.errMsg.errGettingBookmarks=Error getting Internet Explorer Bookmarks.
ExtractIE.getBookmark.errMsg.errPostingBookmarks=Error posting Internet Explorer Bookmark artifacts.
ExtractIE.getCookie.errMsg.errPostinCookiess=Error posting Internet Explorer Cookie artifacts.
ExtractIE.getCookie.errMsg.errPostingCookiess=Error posting Internet Explorer Cookie artifacts.
ExtractIE.getHistory.errMsg.errPostingHistory=Error posting Internet Explorer History artifacts.
ExtractIE.getHistory.errMsg.errPostinHistory=Error posting Internet Explorer History artifacts.
ExtractIE.parentModuleName.noSpace=RecentActivity
Extractor.errPostingArtifacts=Error posting {0} artifacts to the blackboard.
ExtractOs.androidOs.label=Android
ExtractOs.androidVolume.label=OS Drive (Android)
@ -38,6 +50,11 @@ ExtractOs.unitedLinuxVolume.label=OS Drive (Linux United Linux)
ExtractOs.windowsVolume.label=OS Drive (Windows)
ExtractOs.yellowDogLinuxOs.label=Linux (Yellow Dog)
ExtractOs.yellowDogLinuxVolume.label=OS Drive (Linux Yellow Dog)
ExtractOS_progressMessage=Checking for OS
ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files.
ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files
ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files
ExtractSafari_Module_Name=Safari
OpenIDE-Module-Display-Category=Ingest Module
OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\nThe module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web acitivity (sites visited, stored cookies, bookmarked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\nThe module currently supports Windows only disk images. \nThe plugin is also fully functional when deployed on Windows version of Autopsy.
OpenIDE-Module-Name=RecentActivity
@ -62,8 +79,7 @@ Chrome.getAutofill.errMsg.errGettingFiles=Error when trying to get Chrome Web Da
Chrome.getAutofill.errMsg.errAnalyzingFiles={0}: Error while trying to analyze file:{1}
Extract.dbConn.errMsg.failedToQueryDb={0}: Failed to query database.
ExtractIE.moduleName.text=Internet Explorer
ExtractIE.getBookmark.errMsg.errGettingBookmarks={0}: Error getting Internet Explorer Bookmarks.
ExtractIE.parentModuleName.noSpace=RecentActivity
ExtractIE.getBookmark.ere.noSpace=RecentActivity
ExtractIE.parentModuleName=Recent Activity
ExtractIE.getURLFromIEBmkFile.errMsg={0}: Error parsing IE bookmark File {1}
ExtractIE.getURLFromIEBmkFile.errMsg2={0}: Error parsing IE bookmark File {1}
@ -110,6 +126,37 @@ Firefox.getDlPre24.errMsg.errParsingArtifacts={0}: Error parsing {1} Firefox web
Firefox.getDlV24.errMsg.errFetchFiles=Error fetching 'downloads' files for Firefox.
Firefox.getDlV24.errMsg.errAnalyzeFile={0}: Error while trying to analyze file:{1}
Firefox.getDlV24.errMsg.errParsingArtifacts={0}: Error parsing {1} Firefox web download artifacts.
Progress_Message_Analyze_Registry=Analyzing Registry Files
Progress_Message_Analyze_Usage=Data Sources Usage Analysis
Progress_Message_Chrome_AutoFill=Chrome Auto Fill
Progress_Message_Chrome_Bookmarks=Chrome Bookmarks
Progress_Message_Chrome_Cookies=Chrome Cookies
Progress_Message_Chrome_Downloads=Chrome Downloads
Progress_Message_Chrome_FormHistory=Chrome Form History
Progress_Message_Chrome_History=Chrome History
Progress_Message_Chrome_Logins=Chrome Logins
Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks
Progress_Message_Edge_Cookies=Microsoft Edge Cookies
Progress_Message_Edge_History=Microsoft Edge History
Progress_Message_Extract_Resent_Docs=Recent Documents
Progress_Message_Find_Search_Query=Find Search Queries
Progress_Message_Firefox_AutoFill=Firefox Auto Fill
Progress_Message_Firefox_Bookmarks=Firefox Bookmarks
Progress_Message_Firefox_Cookies=Firefox Cookies
Progress_Message_Firefox_Downloads=Firefox Downloads
Progress_Message_Firefox_FormHistory=Firefox Form History
Progress_Message_Firefox_History=Firefox History
Progress_Message_IE_AutoFill=IE Auto Fill
Progress_Message_IE_Bookmarks=IE Bookmarks
Progress_Message_IE_Cookies=IE Cookies
Progress_Message_IE_Downloads=IE Downloads
Progress_Message_IE_FormHistory=IE Form History
Progress_Message_IE_History=IE History
Progress_Message_IE_Logins=IE Logins
Progress_Message_Safari_Bookmarks=Safari Bookmarks
Progress_Message_Safari_Cookies=Safari Cookies
Progress_Message_Safari_Downloads=Safari Downloads
Progress_Message_Safari_History=Safari History
RAImageIngestModule.process.started=Started {0}
RAImageIngestModule.process.errModFailed={0} failed - see log for details <br>
RAImageIngestModule.process.errModErrs={0} had errors -- see log
@ -133,7 +180,6 @@ RecentDocumentsByLnk.parentModuleName.noSpace=RecentActivity
RecentDocumentsByLnk.parentModuleName=Recent Activity
RegRipperFullNotFound=Full version RegRipper executable not found.
RegRipperNotFound=Autopsy RegRipper executable not found.
# {0} - file name
SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}.
SearchEngineURLQueryAnalyzer.moduleName.text=Search Engine
SearchEngineURLQueryAnalyzer.engineName.none=NONE

135
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java
View File

@ -37,11 +37,13 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.datamodel.AbstractFile;
@ -80,28 +82,50 @@ class Chrome extends Extract {
private final Logger logger = Logger.getLogger(this.getClass().getName());
private Content dataSource;
private IngestJobContext context;
private final String moduleName;
@Messages({
"Progress_Message_Chrome_History=Chrome History",
"Progress_Message_Chrome_Bookmarks=Chrome Bookmarks",
"Progress_Message_Chrome_Cookies=Chrome Cookies",
"Progress_Message_Chrome_Downloads=Chrome Downloads",
"Progress_Message_Chrome_FormHistory=Chrome Form History",
"Progress_Message_Chrome_AutoFill=Chrome Auto Fill",
"Progress_Message_Chrome_Logins=Chrome Logins",
})
Chrome() {
moduleName = NbBundle.getMessage(Chrome.class, "Chrome.moduleName");
}
@Override
protected String getName() {
return moduleName;
}
@Override
public void process(Content dataSource, IngestJobContext context) {
public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
this.dataSource = dataSource;
this.context = context;
dataFound = false;
progressBar.progress(Bundle.Progress_Message_Chrome_History());
this.getHistory();
progressBar.progress(Bundle.Progress_Message_Chrome_Bookmarks());
this.getBookmark();
progressBar.progress(Bundle.Progress_Message_Chrome_Cookies());
this.getCookie();
progressBar.progress(Bundle.Progress_Message_Chrome_Logins());
this.getLogins();
progressBar.progress(Bundle.Progress_Message_Chrome_AutoFill());
this.getAutofill();
progressBar.progress(Bundle.Progress_Message_Chrome_Downloads());
this.getDownload();
ChromeCacheExtractor chromeCacheExtractor = new ChromeCacheExtractor(dataSource, context);
chromeCacheExtractor.getCaches();
}
/**
@ -169,22 +193,22 @@ class Chrome extends Extract {
for (HashMap<String, Object> result : tempList) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("url").toString() != null) ? result.get("url").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("last_visit_time").toString()) / 1000000) - Long.valueOf("11644473600"))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("from_visit").toString() != null) ? result.get("from_visit").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("title").toString() != null) ? result.get("title").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "Chrome.moduleName")));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
(NetworkUtils.extractDomain((result.get("url").toString() != null) ? result.get("url").toString() : "")))); //NON-NLS
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY, historyFile, bbattributes);
@ -311,21 +335,16 @@ class Chrome extends Extract {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
//TODO Revisit usage of deprecated constructor as per TSK-583
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(),
"Chrome.parentModuleName"), url));
RecentActivityExtracterModuleFactory.getModuleName(), url));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
NbBundle.getMessage(this.getClass(),
"Chrome.parentModuleName"), name));
RecentActivityExtracterModuleFactory.getModuleName(), name));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
NbBundle.getMessage(this.getClass(),
"Chrome.parentModuleName"), (date / 1000000) - Long.valueOf("11644473600")));
RecentActivityExtracterModuleFactory.getModuleName(), (date / 1000000) - Long.valueOf("11644473600")));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
NbBundle.getMessage(this.getClass(),
"Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "Chrome.moduleName")));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(),
"Chrome.parentModuleName"), domain));
RecentActivityExtracterModuleFactory.getModuleName(), domain));
bbart.addAttributes(bbattributes);
// index the artifact for keyword search
@ -399,25 +418,25 @@ class Chrome extends Extract {
for (HashMap<String, Object> result : tempList) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("host_key").toString() != null) ? result.get("host_key").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("last_access_utc").toString()) / 1000000) - Long.valueOf("11644473600"))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("name").toString() != null) ? result.get("name").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("value").toString() != null) ? result.get("value").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "Chrome.moduleName")));
String domain = result.get("host_key").toString(); //NON-NLS
domain = domain.replaceFirst("^\\.+(?!$)", "");
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"), domain));
RecentActivityExtracterModuleFactory.getModuleName(), domain));
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_COOKIE, cookiesFile, bbattributes);
if (bbart != null) {
@ -493,7 +512,7 @@ class Chrome extends Extract {
for (HashMap<String, Object> result : tempList) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"), (result.get("full_path").toString()))); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(), (result.get("full_path").toString()))); //NON-NLS
long pathID = Util.findID(dataSource, (result.get("full_path").toString())); //NON-NLS
if (pathID != -1) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH_ID,
@ -501,7 +520,7 @@ class Chrome extends Extract {
"Chrome.parentModuleName"), pathID));
}
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("url").toString() != null) ? result.get("url").toString() : ""))); //NON-NLS
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "Recent Activity", ((result.get("url").toString() != null) ? EscapeUtil.decodeURL(result.get("url").toString()) : "")));
Long time = (Long.valueOf(result.get("start_time").toString()) / 1000000) - Long.valueOf("11644473600"); //NON-NLS
@ -509,12 +528,12 @@ class Chrome extends Extract {
//TODO Revisit usage of deprecated constructor as per TSK-583
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LAST_ACCESSED.getTypeID(), "Recent Activity", "Last Visited", time));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"), time));
RecentActivityExtracterModuleFactory.getModuleName(), time));
String domain = NetworkUtils.extractDomain((result.get("url").toString() != null) ? result.get("url").toString() : ""); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"), domain));
RecentActivityExtracterModuleFactory.getModuleName(), domain));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "Chrome.moduleName")));
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_DOWNLOAD, downloadFile, bbattributes);
@ -584,23 +603,23 @@ class Chrome extends Extract {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("origin_url").toString() != null) ? result.get("origin_url").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("date_created").toString()) / 1000000) - Long.valueOf("11644473600"))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
(NetworkUtils.extractDomain((result.get("origin_url").toString() != null) ? result.get("origin_url").toString() : "")))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("username_value").toString() != null) ? result.get("username_value").toString().replaceAll("'", "''") : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("signon_realm").toString() != null) ? result.get("signon_realm").toString() : ""))); //NON-NLS
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_SERVICE_ACCOUNT, loginDataFile, bbattributes);
@ -710,23 +729,23 @@ class Chrome extends Extract {
((result.get("name").toString() != null) ? result.get("name").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("value").toString() != null) ? result.get("value").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
(Integer.valueOf(result.get("count").toString())))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
RecentActivityExtracterModuleFactory.getModuleName(),
Long.valueOf(result.get("date_created").toString()))); //NON-NLS
// get schema version specific attributes
if (isSchemaV8X) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
Long.valueOf(result.get("date_last_used").toString()))); //NON-NLS
}
RecentActivityExtracterModuleFactory.getModuleName(),
Long.valueOf(result.get("date_last_used").toString()))); //NON-NLS
}
// Add an artifact
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_FORM_AUTOFILL, webDataFile, bbattributes);
@ -821,38 +840,38 @@ class Chrome extends Extract {
full_name = String.join(" ", first_name, middle_name, last_name);
}
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME_PERSON,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
full_name)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
full_name)); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
email_Addr)); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
phone_number)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
phone_number)); //NON-NLS
String locationAddress = String.join(", ", street_address, city, state, zipcode, country_code);
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LOCATION,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
locationAddress)); //NON-NLS
if (date_modified > 0) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_MODIFIED,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
date_modified)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
date_modified)); //NON-NLS
}
if (use_count > 0) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
use_count)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
use_count)); //NON-NLS
}
if (use_date > 0) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"),
use_date)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
use_date)); //NON-NLS
}
// Create artifact

1369
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ChromeCacheExtractor.java Normal file
View File

File diff suppressed because it is too large Load Diff

7
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DataSourceUsageAnalyzer.java
View File

@ -26,6 +26,7 @@ import org.apache.commons.io.FilenameUtils;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
@ -46,12 +47,14 @@ class DataSourceUsageAnalyzer extends Extract {
@Messages({
"# {0} - OS name",
"DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})"
"DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})",
"Progress_Message_Analyze_Usage=Data Sources Usage Analysis",
})
@Override
void process(Content dataSource, IngestJobContext context) {
void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
this.dataSource = dataSource;
try {
progressBar.progress(Bundle.Progress_Message_Analyze_Usage());
createDataSourceUsageArtifacts();
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Failed to check if datasource contained a volume with operating system specific files", ex);

251
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java
View File

@ -22,10 +22,18 @@
*/
package org.sleuthkit.autopsy.recentactivity;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
@ -35,9 +43,17 @@ import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.coreutils.SQLiteDBConnect;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
import org.sleuthkit.datamodel.*;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
@Messages({"Extract.indexError.message=Failed to index artifact for keyword search.",
"Extract.noOpenCase.errMsg=No open case available.",
@ -85,7 +101,7 @@ abstract class Extract {
void configExtractor() throws IngestModuleException {
}
abstract void process(Content dataSource, IngestJobContext context);
abstract void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar);
void complete() {
}
@ -141,15 +157,11 @@ abstract class Extract {
*/
void indexArtifact(BlackboardArtifact bbart) {
try {
Blackboard blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard();
// index the artifact for keyword search
blackboard.postArtifact(bbart, getName());
} catch (Blackboard.BlackboardException ex) {
logger.log(Level.SEVERE, "Unable to index blackboard artifact " + bbart.getDisplayName(), ex); //NON-NLS
MessageNotifyUtil.Notify.error(Bundle.Extract_indexError_message(), bbart.getDisplayName());
} catch (NoCurrentCaseException ex) {
logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
MessageNotifyUtil.Notify.error(Bundle.Extract_noOpenCase_errMsg(), bbart.getDisplayName());
}
}
@ -203,7 +215,232 @@ abstract class Extract {
return list;
}
/**
* Returns the state of foundData
*
* @return
*/
public boolean foundData() {
return dataFound;
}
/**
* Sets the value of foundData
*
* @param foundData
*/
protected void setFoundData(boolean foundData) {
dataFound = foundData;
}
/**
* Returns the current case instance
*
* @return Current case instance
*/
protected Case getCurrentCase() {
return this.currentCase;
}
/**
* Creates a list of attributes for a history artifact.
*
* @param url
* @param accessTime Time url was accessed
* @param referrer referred url
* @param title title of the page
* @param programName module name
* @param domain domain of the url
* @param user user that accessed url
*
* @return List of BlackboardAttributes for giving attributes
*
* @throws TskCoreException
*/
protected Collection<BlackboardAttribute> createHistoryAttribute(String url, Long accessTime,
String referrer, String title, String programName, String domain, String user) throws TskCoreException {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL,
RecentActivityExtracterModuleFactory.getModuleName(),
(url != null) ? url : "")); //NON-NLS
if (accessTime != null) {
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
RecentActivityExtracterModuleFactory.getModuleName(), accessTime));
}
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_REFERRER,
RecentActivityExtracterModuleFactory.getModuleName(),
(referrer != null) ? referrer : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TITLE,
RecentActivityExtracterModuleFactory.getModuleName(),
(title != null) ? title : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME,
RecentActivityExtracterModuleFactory.getModuleName(),
(programName != null) ? programName : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN,
RecentActivityExtracterModuleFactory.getModuleName(),
(domain != null) ? domain : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_NAME,
RecentActivityExtracterModuleFactory.getModuleName(),
(user != null) ? user : "")); //NON-NLS
return bbattributes;
}
/**
* Creates a list of attributes for a cookie.
*
* @param url cookie url
* @param creationTime cookie creation time
* @param name cookie name
* @param value cookie value
* @param programName Name of the module creating the attribute
* @param domain Domain of the URL
*
* @return List of BlackboarAttributes for the passed in attributes
*/
protected Collection<BlackboardAttribute> createCookieAttributes(String url,
Long creationTime, String name, String value, String programName, String domain) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL,
RecentActivityExtracterModuleFactory.getModuleName(),
(url != null) ? url : "")); //NON-NLS
if (creationTime != null) {
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME,
RecentActivityExtracterModuleFactory.getModuleName(), creationTime));
}
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME,
RecentActivityExtracterModuleFactory.getModuleName(),
(name != null) ? name : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_VALUE,
RecentActivityExtracterModuleFactory.getModuleName(),
(value != null) ? value : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME,
RecentActivityExtracterModuleFactory.getModuleName(),
(programName != null) ? programName : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN,
RecentActivityExtracterModuleFactory.getModuleName(),
(domain != null) ? domain : "")); //NON-NLS
return bbattributes;
}
/**
* Creates a list of bookmark attributes from the passed in parameters.
*
* @param url Bookmark url
* @param title Title of the bookmarked page
* @param creationTime Date & time at which the bookmark was created
* @param programName Name of the module creating the attribute
* @param domain The domain of the bookmark's url
*
* @return A collection of bookmark attributes
*/
protected Collection<BlackboardAttribute> createBookmarkAttributes(String url, String title, Long creationTime, String programName, String domain) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL,
RecentActivityExtracterModuleFactory.getModuleName(),
(url != null) ? url : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TITLE,
RecentActivityExtracterModuleFactory.getModuleName(),
(title != null) ? title : "")); //NON-NLS
if (creationTime != null) {
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
RecentActivityExtracterModuleFactory.getModuleName(), creationTime));
}
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME,
RecentActivityExtracterModuleFactory.getModuleName(),
(programName != null) ? programName : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN,
RecentActivityExtracterModuleFactory.getModuleName(),
(domain != null) ? domain : "")); //NON-NLS
return bbattributes;
}
/**
* Creates a list of the attributes of a downloaded file
*
* @param path
* @param url URL of the downloaded file
* @param accessTime Time the download occurred
* @param domain Domain of the URL
* @param programName Name of the module creating the attribute
*
* @return A collection of attributed of a downloaded file
*/
protected Collection<BlackboardAttribute> createDownloadAttributes(String path, Long pathID, String url, Long accessTime, String domain, String programName) {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH,
RecentActivityExtracterModuleFactory.getModuleName(),
(path != null) ? path : "")); //NON-NLS
if (pathID != null && pathID != -1) {
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID,
RecentActivityExtracterModuleFactory.getModuleName(),
pathID));
}
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL,
RecentActivityExtracterModuleFactory.getModuleName(),
(url != null) ? url : "")); //NON-NLS
if (accessTime != null) {
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
RecentActivityExtracterModuleFactory.getModuleName(), accessTime));
}
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN,
RecentActivityExtracterModuleFactory.getModuleName(),
(domain != null) ? domain : "")); //NON-NLS
bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME,
RecentActivityExtracterModuleFactory.getModuleName(),
(programName != null) ? programName : "")); //NON-NLS
return bbattributes;
}
/**
* Create temporary file for the given AbstractFile. The new file will be
* created in the temp directory for the module with a unique file name.
*
* @param context
* @param file
*
* @return Newly created copy of the AbstractFile
*
* @throws IOException
*/
protected File createTemporaryFile(IngestJobContext context, AbstractFile file) throws IOException {
Path tempFilePath = Paths.get(RAImageIngestModule.getRATempPath(
getCurrentCase(), getName()), file.getName() + file.getId() + file.getNameExtension());
java.io.File tempFile = tempFilePath.toFile();
try {
ContentUtils.writeToFile(file, tempFile, context::dataSourceIngestIsCancelled);
} catch (IOException ex) {
throw new IOException("Error writingToFile: " + file, ex); //NON-NLS
}
return tempFile;
}
}

862
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java Executable file
View File

@ -0,0 +1,862 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.recentactivity;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Scanner;
import java.util.logging.Level;
import org.openide.modules.InstalledFileLocator;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.ExecUtil;
import org.sleuthkit.autopsy.coreutils.FileUtil;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Extract the bookmarks, cookies, downloads and history from Microsoft Edge
*/
final class ExtractEdge extends Extract {
private static final Logger LOG = Logger.getLogger(ExtractEdge.class.getName());
private final IngestServices services = IngestServices.getInstance();
private final Path moduleTempResultPath;
private Content dataSource;
private IngestJobContext context;
private HashMap<String, ArrayList<String>> containersTable;
private static final String EDGE = "Edge"; //NON-NLS
private static final String EDGE_KEYWORD_VISIT = "Visited:"; //NON-NLS
private static final String IGNORE_COMMA_IN_QUOTES_REGEX = ",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)"; //NON-NLS
private static final String EDGE_TABLE_TYPE_DOWNLOAD = "iedownload"; //NON-NLS
private static final String EDGE_TABLE_TYPE_HISTORY = "History"; //NON-NLS
private static final String EDGE_TABLE_TYPE_COOKIE = "cookie"; //NON-NLS
private static final String EDGE_HEAD_URL = "url"; //NON-NLS
private static final String EDGE_HEAD_ACCESSTIME = "accessedtime"; //NON-NLS
private static final String EDGE_HEAD_NAME = "name"; //NON-NLS
private static final String EDGE_HEAD_CONTAINER_ID = "containerid"; //NON-NLS
private static final String EDGE_HEAD_RESPONSEHEAD = "responseheaders"; //NON-NLS
private static final String EDGE_HEAD_TITLE = "title"; //NON-NLS
private static final String EDGE_HEAD_RDOMAIN = "rdomain"; //NON-NLS
private static final String EDGE_HEAD_VALUE = "value"; //NON-NLS
private static final String EDGE_HEAD_LASTMOD = "lastmodified"; //NON-NLS
private static final String EDGE_WEBCACHE_PREFIX = "WebCacheV01"; //NON-NLS
private static final String EDGE_CONTAINER_FILE_PREFIX = "Container_"; //NON-NLS
private static final String EDGE_CONTAINER_FILE_EXT = ".csv"; //NON-NLS
private static final String EDGE_WEBCACHE_EXT = ".dat"; //NON-NLS
private static final String ESE_TOOL_NAME = "ESEDatabaseView.exe"; //NON-NLS
private static final String EDGE_WEBCACHE_NAME = "WebCacheV01.dat"; //NON-NLS
private static final String EDGE_SPARTAN_NAME = "Spartan.edb"; //NON-NLS
private static final String EDGE_CONTAINTERS_FILE_NAME = "Containers.csv"; //NON-NLS
private static final String EDGE_FAVORITE_FILE_NAME = "Favorites.csv"; //NON-NLS
private static final String EDGE_OUTPUT_FILE_NAME = "Output.txt"; //NON-NLS
private static final String EDGE_ERROR_FILE_NAME = "File.txt"; //NON-NLS
private static final String EDGE_WEBCACHE_FOLDER_NAME = "WebCache"; //NON-NLS
private static final String EDGE_SPARTAN_FOLDER_NAME = "MicrosoftEdge"; //NON-NLS
private static final String ESE_TOOL_FOLDER = "ESEDatabaseView"; //NON-NLS
private static final String EDGE_RESULT_FOLDER_NAME = "results"; //NON-NLS
private static final SimpleDateFormat DATE_FORMATTER = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a"); //NON-NLS
@Messages({
"ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer",
"ExtractEdge_process_errMsg_errGettingWebCacheFiles=Error trying to retrieving Edge WebCacheV01 file",
"ExtractEdge_process_errMsg_webcacheFail=Failure processing Microsoft Edge WebCacheV01.dat file",
"ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file",
"ExtractEdge_Module_Name=Microsoft Edge",
"ExtractEdge_getHistory_containerFileNotFound=Error while trying to analyze Edge history",
"Progress_Message_Edge_History=Microsoft Edge History",
"Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks",
"Progress_Message_Edge_Cookies=Microsoft Edge Cookies",
})
/**
* Extract the bookmarks, cookies, downloads and history from Microsoft Edge
*/
ExtractEdge() throws NoCurrentCaseException {
moduleTempResultPath = Paths.get(RAImageIngestModule.getRATempPath(Case.getCurrentCaseThrows(), EDGE), EDGE_RESULT_FOLDER_NAME);
}
@Override
protected String getName() {
return Bundle.ExtractEdge_Module_Name();
}
@Override
void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
this.dataSource = dataSource;
this.context = context;
this.setFoundData(false);
List<AbstractFile> webCacheFiles = null;
List<AbstractFile> spartanFiles = null;
try {
webCacheFiles = fetchWebCacheDBFiles();
} catch (TskCoreException ex) {
this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_errGettingWebCacheFiles());
LOG.log(Level.SEVERE, "Error fetching 'WebCacheV01.dat' files for Microsoft Edge", ex); //NON-NLS
}
try {
spartanFiles = fetchSpartanDBFiles(); // For later use with bookmarks
} catch (TskCoreException ex) {
this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail());
LOG.log(Level.SEVERE, "Error fetching 'spartan.edb' files for Microsoft Edge", ex); //NON-NLS
}
// No edge files found
if (webCacheFiles == null && spartanFiles == null) {
return;
}
this.setFoundData(true);
if (!PlatformUtil.isWindowsOS()) {
LOG.log(Level.WARNING, "Microsoft Edge files found, unable to parse on Non-Windows system"); //NON-NLS
return;
}
final String esedumper = getPathForESEDumper();
if (esedumper == null) {
this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_unableFindESEViewer());
LOG.log(Level.SEVERE, "Error finding ESEDatabaseViewer program"); //NON-NLS
return; //If we cannot find the ESEDatabaseView we cannot proceed
}
try {
this.processWebCacheDbFile(esedumper, webCacheFiles, progressBar);
} catch (IOException | TskCoreException ex) {
this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail());
LOG.log(Level.SEVERE, "Error returned from processWebCacheDbFile", ex); // NON-NLS
}
progressBar.progress(Bundle.Progress_Message_Edge_Bookmarks());
try {
this.processSpartanDbFile(esedumper, spartanFiles);
} catch (IOException | TskCoreException ex) {
this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail());
LOG.log(Level.SEVERE, "Error returned from processSpartanDbFile", ex); // NON-NLS
}
}
/**
* Process WebCacheV01.dat ese database file creating artifacts for cookies,
* and history contained within.
*
* @param eseDumperPath Path to ESEDatabaseView.exe
* @param webCacheFiles List of case WebCacheV01.dat files
* @throws IOException
* @throws TskCoreException
*/
void processWebCacheDbFile(String eseDumperPath, List<AbstractFile> webCacheFiles, DataSourceIngestModuleProgress progressBar) throws IOException, TskCoreException {
for (AbstractFile webCacheFile : webCacheFiles) {
if (context.dataSourceIngestIsCancelled()) {
return;
}
clearContainerTable();
//Run the dumper
String tempWebCacheFileName = EDGE_WEBCACHE_PREFIX
+ Integer.toString((int) webCacheFile.getId()) + EDGE_WEBCACHE_EXT; //NON-NLS
File tempWebCacheFile = new File(RAImageIngestModule.getRATempPath(currentCase, EDGE), tempWebCacheFileName);
try {
ContentUtils.writeToFile(webCacheFile, tempWebCacheFile,
context::dataSourceIngestIsCancelled);
} catch (IOException ex) {
throw new IOException("Error writingToFile: " + webCacheFile, ex); //NON-NLS
}
File resultsDir = new File(moduleTempResultPath.toAbsolutePath() + Integer.toString((int) webCacheFile.getId()));
resultsDir.mkdirs();
try {
executeDumper(eseDumperPath, tempWebCacheFile.getAbsolutePath(),
resultsDir.getAbsolutePath());
if (context.dataSourceIngestIsCancelled()) {
return;
}
progressBar.progress(Bundle.Progress_Message_Edge_History());
this.getHistory(webCacheFile, resultsDir);
if (context.dataSourceIngestIsCancelled()) {
return;
}
progressBar.progress(Bundle.Progress_Message_Edge_Cookies());
this.getCookies(webCacheFile, resultsDir);
} finally {
tempWebCacheFile.delete();
FileUtil.deleteFileDir(resultsDir);
}
}
}
/**
* Process spartan.edb ese database file creating artifacts for the bookmarks
* contained within.
*
* @param eseDumperPath Path to ESEDatabaseViewer
* @param spartanFiles List of the case spartan.edb files
* @throws IOException
* @throws TskCoreException
*/
void processSpartanDbFile(String eseDumperPath, List<AbstractFile> spartanFiles) throws IOException, TskCoreException {
for (AbstractFile spartanFile : spartanFiles) {
if (context.dataSourceIngestIsCancelled()) {
return;
}
//Run the dumper
String tempSpartanFileName = EDGE_WEBCACHE_PREFIX
+ Integer.toString((int) spartanFile.getId()) + EDGE_WEBCACHE_EXT;
File tempSpartanFile = new File(RAImageIngestModule.getRATempPath(currentCase, EDGE), tempSpartanFileName);
try {
ContentUtils.writeToFile(spartanFile, tempSpartanFile,
context::dataSourceIngestIsCancelled);
} catch (IOException ex) {
throw new IOException("Error writingToFile: " + spartanFile, ex); //NON-NLS
}
File resultsDir = new File(moduleTempResultPath.toAbsolutePath() + Integer.toString((int) spartanFile.getId()));
resultsDir.mkdirs();
try {
executeDumper(eseDumperPath, tempSpartanFile.getAbsolutePath(),
resultsDir.getAbsolutePath());
if (context.dataSourceIngestIsCancelled()) {
return;
}
this.getBookmarks(spartanFile, resultsDir);
} finally {
tempSpartanFile.delete();
FileUtil.deleteFileDir(resultsDir);
}
}
}
/**
* getHistory searches the files with "container" in the file name for lines
* with the text "Visited" in them. Note that not all of the container
* files, if fact most of them do not, have the browser history in them.
* @param origFile Original case file
* @param resultDir Output directory of ESEDatabaseViewer
* @throws TskCoreException
* @throws FileNotFoundException
*/
private void getHistory(AbstractFile origFile, File resultDir) throws TskCoreException, FileNotFoundException {
ArrayList<File> historyFiles = getHistoryFiles(resultDir);
if (historyFiles == null) {
return;
}
for (File file : historyFiles) {
Scanner fileScanner;
try {
fileScanner = new Scanner(new FileInputStream(file.toString()));
} catch (FileNotFoundException ex) {
LOG.log(Level.WARNING, "Unable to find the ESEDatabaseView file at " + file.getPath(), ex); //NON-NLS
continue; // If we couldn't open this file, continue to the next file
}
Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
try {
List<String> headers = null;
while (fileScanner.hasNext()) {
String line = fileScanner.nextLine();
if (headers == null) {
headers = Arrays.asList(line.toLowerCase().split(","));
continue;
}
if (line.contains(EDGE_KEYWORD_VISIT)) {
BlackboardArtifact ba = getHistoryArtifact(origFile, headers, line);
if (ba != null) {
bbartifacts.add(ba);
this.indexArtifact(ba);
}
}
}
} finally {
fileScanner.close();
}
if (!bbartifacts.isEmpty()) {
services.fireModuleDataEvent(new ModuleDataEvent(
RecentActivityExtracterModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
}
}
}
/**
* Search for bookmark files and make artifacts.
*
* @param origFile Original case file
* @param resultDir Output directory of ESEDatabaseViewer
* @throws TskCoreException
* @throws FileNotFoundException
*/
private void getBookmarks(AbstractFile origFile, File resultDir) throws TskCoreException {
Scanner fileScanner;
File favoriteFile = new File(resultDir, EDGE_FAVORITE_FILE_NAME);
try {
fileScanner = new Scanner(new FileInputStream(favoriteFile));
} catch (FileNotFoundException ex) {
// This is a non-fatal error, if the favorites file is not found
// there might have not been any favorites\bookmarks
return;
}
Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
try {
List<String> headers = null;
while (fileScanner.hasNext()) {
String line = fileScanner.nextLine();
if (headers == null) {
headers = Arrays.asList(line.toLowerCase().split(","));
continue;
}
BlackboardArtifact ba = getBookmarkArtifact(origFile, headers, line);
if (ba != null) {
bbartifacts.add(ba);
this.indexArtifact(ba);
}
}
} finally {
fileScanner.close();
}
if (!bbartifacts.isEmpty()) {
services.fireModuleDataEvent(new ModuleDataEvent(
RecentActivityExtracterModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
}
}
/**
* Queries for cookie files and adds artifacts.
*
* @param origFile Original case file
* @param resultDir Output directory of ESEDatabaseViewer
* @throws TskCoreException
*/
private void getCookies(AbstractFile origFile, File resultDir) throws TskCoreException {
File containerFiles[] = resultDir.listFiles((dir, name) -> name.toLowerCase().contains(EDGE_TABLE_TYPE_COOKIE));
if (containerFiles == null) {
return;
}
for (File file : containerFiles) {
Scanner fileScanner;
try {
fileScanner = new Scanner(new FileInputStream(file.toString()));
} catch (FileNotFoundException ex) {
LOG.log(Level.WARNING, "Unable to find the ESEDatabaseView file at " + file.getPath(), ex); //NON-NLS
continue; // If we couldn't open this file, continue to the next file
}
Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
try {
List<String> headers = null;
while (fileScanner.hasNext()) {
String line = fileScanner.nextLine();
if (headers == null) {
headers = Arrays.asList(line.toLowerCase().split(","));
continue;
}
BlackboardArtifact ba = getCookieArtifact(origFile, headers, line);
if (ba != null) {
bbartifacts.add(ba);
this.indexArtifact(ba);
}
}
} finally {
fileScanner.close();
}
if (!bbartifacts.isEmpty()) {
services.fireModuleDataEvent(new ModuleDataEvent(
RecentActivityExtracterModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
}
}
}
/**
* Queries for download files and adds artifacts.
*
* Leaving for future use.
*
* @param origFile Original case file
* @param resultDir Output directory of ESEDatabaseViewer
* @throws TskCoreException
* @throws FileNotFoundException
*/
private void getDownloads(AbstractFile origFile, File resultDir) throws TskCoreException, FileNotFoundException {
ArrayList<File> downloadFiles = getDownloadFiles(resultDir);
if (downloadFiles == null) {
return;
}
for (File file : downloadFiles) {
Scanner fileScanner;
try {
fileScanner = new Scanner(new FileInputStream(file.toString()));
} catch (FileNotFoundException ex) {
LOG.log(Level.WARNING, "Unable to find the ESEDatabaseView file at " + file.getPath(), ex); //NON-NLS
continue; // If we couldn't open this file, continue to the next file
}
Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
try {
List<String> headers = null;
while (fileScanner.hasNext()) {
String line = fileScanner.nextLine();
if (headers == null) {
headers = Arrays.asList(line.toLowerCase().split(","));
continue;
}
if (line.contains(EDGE_TABLE_TYPE_DOWNLOAD)) {
BlackboardArtifact ba = getDownloadArtifact(origFile, headers, line);
if (ba != null) {
bbartifacts.add(ba);
this.indexArtifact(ba);
}
}
}
} finally {
fileScanner.close();
}
if (!bbartifacts.isEmpty()) {
services.fireModuleDataEvent(new ModuleDataEvent(
RecentActivityExtracterModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD, bbartifacts));
}
}
}
/**
* Find the location of ESEDatabaseViewer.exe
*
* @return Absolute path to ESEDatabaseViewer.exe or null if the file is not found
*/
private String getPathForESEDumper() {
Path path = Paths.get(ESE_TOOL_FOLDER, ESE_TOOL_NAME);
File eseToolFile = InstalledFileLocator.getDefault().locate(path.toString(),
ExtractEdge.class.getPackage().getName(), false);
if (eseToolFile != null) {
return eseToolFile.getAbsolutePath();
}
return null;
}
/**
* Finds all of the WebCacheV01.dat files in the case
*
* @return A list of WebCacheV01.dat files, possibly empty if none are found
* @throws TskCoreException
*/
private List<AbstractFile> fetchWebCacheDBFiles() throws TskCoreException {
org.sleuthkit.autopsy.casemodule.services.FileManager fileManager
= currentCase.getServices().getFileManager();
return fileManager.findFiles(dataSource, EDGE_WEBCACHE_NAME, EDGE_WEBCACHE_FOLDER_NAME);
}
/**
* Finds all of the spartan.edb files in the case
*
* @return A list of spartan files, possibly empty if none are found
* @throws TskCoreException
*/
private List<AbstractFile> fetchSpartanDBFiles() throws TskCoreException {
org.sleuthkit.autopsy.casemodule.services.FileManager fileManager
= currentCase.getServices().getFileManager();
return fileManager.findFiles(dataSource, EDGE_SPARTAN_NAME, EDGE_SPARTAN_FOLDER_NAME);
}
/**
* Executes the ESEViewDumper on the given inputFile.
*
* Each table in the ese database will be dumped as a comma separated file
* named <tableName>.csv
*
* @param dumperPath Path to ESEDatabaseView.exe
* @param inputFilePath Path to ese database file to be dumped
* @param outputDir Output directory for dumper
* @throws IOException
*/
private void executeDumper(String dumperPath, String inputFilePath,
String outputDir) throws IOException {
final Path outputFilePath = Paths.get(outputDir, EDGE_OUTPUT_FILE_NAME);
final Path errFilePath = Paths.get(outputDir, EDGE_ERROR_FILE_NAME);
LOG.log(Level.INFO, "Writing ESEDatabaseViewer results to: {0}", outputDir); //NON-NLS
List<String> commandLine = new ArrayList<>();
commandLine.add(dumperPath);
commandLine.add("/table"); //NON-NLS
commandLine.add(inputFilePath);
commandLine.add("*"); //NON-NLS
commandLine.add("/scomma"); //NON-NLS
commandLine.add(outputDir + "\\" + "*.csv"); //NON-NLS
ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
processBuilder.redirectOutput(outputFilePath.toFile());
processBuilder.redirectError(errFilePath.toFile());
ExecUtil.execute(processBuilder, new DataSourceIngestModuleProcessTerminator(context));
}
/**
* Create a BlackboardArtifact for the given row from the Edge history
* table.
*
* @param origFile Original case file
* @param headers List of table headers
* @param line CSV string representing a row of history table
* @return BlackboardArtifact representing one history table entry
* @throws TskCoreException
*/
private BlackboardArtifact getHistoryArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
String[] rowSplit = line.split(",");
int index = headers.indexOf(EDGE_HEAD_URL);
String urlUserStr = rowSplit[index];
String[] str = urlUserStr.split("@");
String user = (str[0].replace(EDGE_KEYWORD_VISIT, "")).trim();
String url = str[1];
index = headers.indexOf(EDGE_HEAD_ACCESSTIME);
String accessTime = rowSplit[index].trim();
Long ftime = null;
try {
Long epochtime = DATE_FORMATTER.parse(accessTime).getTime();
ftime = epochtime / 1000;
} catch (ParseException ex) {
LOG.log(Level.WARNING, "The Accessed Time format in history file seems invalid " + accessTime, ex); //NON-NLS
}
BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY);
bbart.addAttributes(createHistoryAttribute(url, ftime,
null, null,
this.getName(),
NetworkUtils.extractDomain(url), user));
return bbart;
}
/**
* Create a BlackboardArtifact for the given row from the Edge cookie table.
*
* @param origFile Original case file
* @param headers List of table headers
* @param line CSV string representing a row of cookie table
* @return BlackboardArtifact representing one cookie table entry
* @throws TskCoreException
*/
private BlackboardArtifact getCookieArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
String[] lineSplit = line.split(","); // NON-NLS
String accessTime = lineSplit[headers.indexOf(EDGE_HEAD_LASTMOD)].trim();
Long ftime = null;
try {
Long epochtime = DATE_FORMATTER.parse(accessTime).getTime();
ftime = epochtime / 1000;
} catch (ParseException ex) {
LOG.log(Level.WARNING, "The Accessed Time format in history file seems invalid " + accessTime, ex); //NON-NLS
}
String domain = lineSplit[headers.indexOf(EDGE_HEAD_RDOMAIN)].trim();
String name = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_NAME)].trim());
String value = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_VALUE)].trim());
String url = flipDomain(domain);
BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE);
bbart.addAttributes(createCookieAttributes(url, ftime, name, value, this.getName(), NetworkUtils.extractDomain(url)));
return bbart;
}
/**
* Create a BlackboardArtifact for the given row from the Edge cookie table.
*
* This function is on hold for the moment. All of the information need
* seems to be in decodedheader, but its not currently obvious how to pull
* it apart.
*
* @param origFile Original case file
* @param headers List of table headers
* @param line CSV string representing a row of download table
* @return BlackboardArtifact representing one download table entry
* @throws TskCoreException
*/
private BlackboardArtifact getDownloadArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
BlackboardArtifact bbart = null;
String[] lineSplit = line.split(","); // NON-NLS
String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)];
return bbart;
}
/**
* Parse the comma separated row of information from the "Favorites" table
* of the spartan database.
*
* Note: The "Favorites" table does not have a "Creation Time"
*
* @param origFile File the table came from ie spartan.edb
* @param headers List of table column headers
* @param line The line or row of the table to parse
* @return BlackboardArtifact representation of the passed in line\table row or null if no Bookmark is found
* @throws TskCoreException
*/
private BlackboardArtifact getBookmarkArtifact(AbstractFile origFile, List<String> headers, String line) throws TskCoreException {
// split on all commas as long as they are not inbetween quotes
String[] lineSplit = line.split(IGNORE_COMMA_IN_QUOTES_REGEX, -1);
String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)];
String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace("\"", ""); // NON-NLS
if (url.isEmpty()) {
return null;
}
BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
bbart.addAttributes(createBookmarkAttributes(url, title, null,
this.getName(), NetworkUtils.extractDomain(url)));
return bbart;
}
/**
* Converts a space separated string of hex values to ascii characters.
*
* @param hexString
* @return "decoded" string or null if a non-hex value was found
*/
private String hexToChar(String hexString) {
String[] hexValues = hexString.split(" "); // NON-NLS
StringBuilder output = new StringBuilder();
for (String str : hexValues) {
try {
int value = Integer.parseInt(str, 16);
if (value > 31) { // Ignore non-print characters
output.append((char) value);
}
} catch (NumberFormatException ex) {
return null;
}
}
return output.toString();
}
/**
* The RDomain in the WebCacheV01.data cookies tables are backwards, this
* function corrects them.
*
* Values in the RDomain appear as either com.microsoft.www or com.microsoft
* but for some reason there could also be "junk". the length checks are
* there to weed out the "junk".
*
* @param domain
* @return Correct domain string
*/
private String flipDomain(String domain) {
if (domain == null || domain.isEmpty()) {
return null;
}
String[] tokens = domain.split("\\."); // NON-NLS
if (tokens.length < 2 || tokens.length > 3) {
return domain; // don't know what to do, just send it back as is
}
StringBuilder buf = new StringBuilder();
if (tokens.length > 2) {
buf.append(tokens[2]);
buf.append(".");
}
buf.append(tokens[1]);
buf.append(".");
buf.append(tokens[0]);
return buf.toString();
}
/**
* Returns a list the container files that have download information in
* them.
*
* @param resultDir Path to ESEDatabaseViewer output
* @return List of download table files
*/
private ArrayList<File> getDownloadFiles(File resultDir) throws FileNotFoundException {
return getContainerFiles(resultDir, EDGE_TABLE_TYPE_DOWNLOAD);
}
/**
* Returns a list the container files that have history information in them.
*
* @param resultDir Path to ESEDatabaseViewer output
* @return List of history table files
* @throws FileNotFoundException
*/
private ArrayList<File> getHistoryFiles(File resultDir) throws FileNotFoundException {
return getContainerFiles(resultDir, EDGE_TABLE_TYPE_HISTORY);
}
/**
* Returns a list of the containers files that are of the given type string
*
* @param resultDir Path to ESEDatabaseViewer output
* @param type Type of table files
* @return List of table files returns null if no files of that type are found
* @throws FileNotFoundException
*/
private ArrayList<File> getContainerFiles(File resultDir, String type) throws FileNotFoundException {
HashMap<String, ArrayList<String>> idTable = getContainerIDTable(resultDir);
ArrayList<String> idList = idTable.get(type);
if (idList == null) {
return null;
}
ArrayList<File> fileList = new ArrayList<>();
for (String str : idList) {
String fileName = EDGE_CONTAINER_FILE_PREFIX + str + EDGE_CONTAINER_FILE_EXT;
fileList.add(new File(resultDir, fileName));
}
return fileList;
}
/**
* Opens and reads the Containers table to create a table of information
* about which of the Continer_xx files contain which type of information.
*
* Each row of the "Containers" table describes one of the Container_xx
* files.
*
* @param resultDir Path to ESEDatabaseViewer output
* @return Hashmap with Key representing the table type, the value is a list of table ids for that type
*/
private HashMap<String, ArrayList<String>> getContainerIDTable(File resultDir) throws FileNotFoundException {
if (containersTable == null) {
File containerFile = new File(resultDir, EDGE_CONTAINTERS_FILE_NAME);
try (Scanner fileScanner = new Scanner(new FileInputStream(containerFile))) {
List<String> headers = null;
containersTable = new HashMap<>();
int nameIdx = 0;
int idIdx = 0;
while (fileScanner.hasNext()) {
String line = fileScanner.nextLine();
if (headers == null) {
headers = Arrays.asList(line.toLowerCase().split(","));
nameIdx = headers.indexOf(EDGE_HEAD_NAME);
idIdx = headers.indexOf(EDGE_HEAD_CONTAINER_ID);
} else {
String[] row = line.split(","); // NON-NLS
String name = row[nameIdx];
String id = row[idIdx];
ArrayList<String> idList = containersTable.get(name);
if (idList == null) {
idList = new ArrayList<>();
containersTable.put(name, idList);
}
idList.add(id);
}
}
}
}
return containersTable;
}
/**
* Clears the containerTable
*/
private void clearContainerTable(){
containersTable = null;
}
}

572
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java
View File

@ -22,9 +22,6 @@
*/
package org.sleuthkit.autopsy.recentactivity;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@ -34,17 +31,15 @@ import java.io.InputStreamReader;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.Scanner;
import java.util.Set;
import java.util.logging.Level;
import javax.annotation.concurrent.Immutable;
import org.apache.commons.lang3.StringUtils;
import java.util.stream.Collectors;
import org.openide.modules.InstalledFileLocator;
import org.openide.util.Exceptions;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.ExecUtil;
@ -53,29 +48,14 @@ import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.datamodel.*;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_OS_ACCOUNT;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY;
import org.sleuthkit.datamodel.BlackboardAttribute;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_REFERRER;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TITLE;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_NAME;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_VALUE;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import static org.sleuthkit.autopsy.recentactivity.Bundle.*;
/**
* Extracts activity from Internet Explorer browser, as well as recent documents
@ -84,47 +64,59 @@ import org.sleuthkit.datamodel.TskCoreException;
class ExtractIE extends Extract {
private static final Logger logger = Logger.getLogger(ExtractIE.class.getName());
private static final String PARENT_MODULE_NAME
= NbBundle.getMessage(ExtractIE.class, "ExtractIE.parentModuleName.noSpace");
private static final String PASCO_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
private static final String JAVA_PATH = PlatformUtil.getJavaPath();
private final String moduleTempResultsDir;
private String PASCO_LIB_PATH;
private final String JAVA_PATH;
private static final String RESOURCE_URL_PREFIX = "res://";
private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
private Content dataSource;
private IngestJobContext context;
@Messages({
"Progress_Message_IE_History=IE History",
"Progress_Message_IE_Bookmarks=IE Bookmarks",
"Progress_Message_IE_Cookies=IE Cookies",
"Progress_Message_IE_Downloads=IE Downloads",
"Progress_Message_IE_FormHistory=IE Form History",
"Progress_Message_IE_AutoFill=IE Auto Fill",
"Progress_Message_IE_Logins=IE Logins",})
ExtractIE() throws NoCurrentCaseException {
moduleName = NbBundle.getMessage(ExtractIE.class, "ExtractIE.moduleName.text");
moduleTempResultsDir = RAImageIngestModule.getRATempPath(Case.getCurrentCaseThrows(), "IE") + File.separator + "results"; //NON-NLS
JAVA_PATH = PlatformUtil.getJavaPath();
}
@Override
protected String getName() {
return NbBundle.getMessage(ExtractIE.class, "ExtractIE.moduleName.text");
}
@Override
public void process(Content dataSource, IngestJobContext context) {
public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
this.dataSource = dataSource;
this.context = context;
dataFound = false;
progressBar.progress(Bundle.Progress_Message_IE_Bookmarks());
this.getBookmark();
progressBar.progress(Bundle.Progress_Message_IE_Cookies());
this.getCookie();
progressBar.progress(Bundle.Progress_Message_IE_History());
this.getHistory();
}
/**
* Finds the files storing bookmarks and creates artifacts
*/
@Messages({"ExtractIE.getBookmark.errMsg.errGettingBookmarks=Error getting Internet Explorer Bookmarks.",
"ExtractIE.getBookmark.errMsg.errPostingBookmarks=Error posting Internet Explorer Bookmark artifacts."
})
private void getBookmark() {
List<AbstractFile> favoritesFiles;
try {
favoritesFiles = fileManager.findFiles(dataSource, "%.url", "Favorites"); //NON-NLS
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Error fetching 'url' files for Internet Explorer bookmarks.", ex); //NON-NLS
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.getBookmark.errMsg.errGettingBookmarks",
this.getName()));
this.addErrorMessage(Bundle.ExtractIE_getBookmark_errMsg_errGettingBookmarks());
return;
}
@ -144,70 +136,91 @@ class ExtractIE extends Extract {
break;
}
Collection<BlackboardAttribute> bbattributes = Arrays.asList(
new BlackboardAttribute(
TSK_URL, PARENT_MODULE_NAME, getURLFromIEBookmarkFile(fav)),
new BlackboardAttribute(
TSK_TITLE, PARENT_MODULE_NAME, fav.getName()),
new BlackboardAttribute(
TSK_DATETIME_CREATED, PARENT_MODULE_NAME, fav.getCrtime()),
new BlackboardAttribute(
TSK_PROG_NAME, PARENT_MODULE_NAME,
NbBundle.getMessage(this.getClass(), "ExtractIE.moduleName.text")),
new BlackboardAttribute(
TSK_DOMAIN, PARENT_MODULE_NAME, NetworkUtils.extractDomain(getURLFromIEBookmarkFile(fav))));
try {
BlackboardArtifact bbart = fav.newArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
bbart.addAttributes(bbattributes);
String url = getURLFromIEBookmarkFile(fav);
String name = fav.getName();
Long datetime = fav.getCrtime();
String Tempdate = datetime.toString();
datetime = Long.valueOf(Tempdate);
String domain = extractDomain(url);
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
RecentActivityExtracterModuleFactory.getModuleName(), url));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
RecentActivityExtracterModuleFactory.getModuleName(), name));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
RecentActivityExtracterModuleFactory.getModuleName(), datetime));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "ExtractIE.moduleName.text")));
if (domain != null && domain.isEmpty() == false) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
RecentActivityExtracterModuleFactory.getModuleName(), domain));
}
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK, fav, bbattributes);
if (bbart != null) {
bbartifacts.add(bbart);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error while trying to create Internet Explorer bookmark artifact.", ex); //NON-NLS
this.addErrorMessage(NbBundle.getMessage(Chrome.class, "ExtractIE.getBookmark.errMsg.errGettingBookmarks", //NON-NLS
this.getName(), fav.getName()));
}
}
try {
blackboard.postArtifacts(bbartifacts, PARENT_MODULE_NAME);
blackboard.postArtifacts(bbartifacts, moduleName);
} catch (Blackboard.BlackboardException ex) {
logger.log(Level.SEVERE, "Error while trying to post Internet Explorer bookmark artifact.", ex); //NON-NLS
this.addErrorMessage(Bundle.Extractor_errPostingArtifacts(getName()));
this.addErrorMessage(Bundle.ExtractIE_getBookmark_errMsg_errPostingBookmarks());
logger.log(Level.SEVERE, "Exception thrown while posting IE bookmark artifact.", ex); //NON-NLS
}
}
private String getURLFromIEBookmarkFile(AbstractFile fav) {
String line;
try (BufferedReader reader = new BufferedReader(new InputStreamReader(new ReadContentInputStream(fav)));) {
while (null != (line = reader.readLine())) {
BufferedReader reader = new BufferedReader(new InputStreamReader(new ReadContentInputStream(fav)));
String line, url = "";
try {
line = reader.readLine();
while (null != line) {
// The actual shortcut line we are interested in is of the
// form URL=http://path/to/website
if (line.startsWith("URL")) { //NON-NLS
return StringUtils.substringAfter(line, "="); //NON-NLS
url = line.substring(line.indexOf("=") + 1);
break;
}
line = reader.readLine();
}
} catch (IOException ex) {
logger.log(Level.WARNING, "Failed to read from content: " + fav.getName(), ex); //NON-NLS
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.getURLFromIEBmkFile.errMsg", this.getName(),
fav.getName()));
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.getURLFromIEBmkFile.errMsg", this.getName(),
fav.getName()));
} catch (IndexOutOfBoundsException ex) {
logger.log(Level.WARNING, "Failed while getting URL of IE bookmark. Unexpected format of the bookmark file: " + fav.getName(), ex); //NON-NLS
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.getURLFromIEBmkFile.errMsg2", this.getName(),
fav.getName()));
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.getURLFromIEBmkFile.errMsg2", this.getName(),
fav.getName()));
} finally {
try {
reader.close();
} catch (IOException ex) {
logger.log(Level.WARNING, "Failed to close reader.", ex); //NON-NLS
}
}
return "";
return url;
}
/**
* Finds files that store cookies and adds artifacts for them.
*/
@Messages({
"ExtractIE.getCookie.errMsg.errPostingCookiess=Error posting Internet Explorer Cookie artifacts."
})
private void getCookie() {
List<AbstractFile> cookiesFiles;
try {
cookiesFiles = fileManager.findFiles(dataSource, "%.txt", "Cookies"); //NON-NLS
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Error getting cookie files for IE"); //NON-NLS
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.getCookie.errMsg.errGettingFile", this.getName()));
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.getCookie.errMsg.errGettingFile", this.getName()));
return;
}
@ -226,65 +239,69 @@ class ExtractIE extends Extract {
continue;
}
byte[] cookiesBuffer = new byte[(int) cookiesFile.getSize()];
byte[] t = new byte[(int) cookiesFile.getSize()];
try {
cookiesFile.read(cookiesBuffer, 0, cookiesFile.getSize());
final int bytesRead = cookiesFile.read(t, 0, cookiesFile.getSize());
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Error reading bytes of Internet Explorer cookie.", ex); //NON-NLS
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.getCookie.errMsg.errReadingIECookie",
this.getName(), cookiesFile.getName()));
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.getCookie.errMsg.errReadingIECookie",
this.getName(), cookiesFile.getName()));
continue;
}
String cookieString = new String(t);
String[] values = cookieString.split("\n");
String url = values.length > 2 ? values[2] : "";
String value = values.length > 1 ? values[1] : "";
String name = values.length > 0 ? values[0] : "";
Long datetime = cookiesFile.getCrtime();
String tempDate = datetime.toString();
datetime = Long.valueOf(tempDate);
String domain = extractDomain(url);
String[] values = new String(cookiesBuffer).split("\n");
String URL = values.length > 2 ? values[2] : "";
Collection<BlackboardAttribute> bbattributes = Arrays.asList(new BlackboardAttribute(
TSK_DATETIME, PARENT_MODULE_NAME,
cookiesFile.getCrtime()),
new BlackboardAttribute(
TSK_NAME, PARENT_MODULE_NAME,
values.length > 0 ? values[0] : ""),
new BlackboardAttribute(
TSK_VALUE, PARENT_MODULE_NAME,
values.length > 1 ? values[1] : ""),
new BlackboardAttribute(
TSK_URL, PARENT_MODULE_NAME,
URL),
new BlackboardAttribute(
TSK_PROG_NAME, PARENT_MODULE_NAME,
getName()),
new BlackboardAttribute(
TSK_DOMAIN, PARENT_MODULE_NAME,
NetworkUtils.extractDomain(URL)));
try {
BlackboardArtifact bbart = cookiesFile.newArtifact(TSK_WEB_COOKIE);
bbart.addAttributes(bbattributes);
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
RecentActivityExtracterModuleFactory.getModuleName(), url));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME,
RecentActivityExtracterModuleFactory.getModuleName(), datetime));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
RecentActivityExtracterModuleFactory.getModuleName(), (name != null) ? name : ""));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
RecentActivityExtracterModuleFactory.getModuleName(), value));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "ExtractIE.moduleName.text")));
if (domain != null && domain.isEmpty() == false) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
RecentActivityExtracterModuleFactory.getModuleName(), domain));
}
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_COOKIE, cookiesFile, bbattributes);
if (bbart != null) {
bbartifacts.add(bbart);
} catch (TskCoreException ex) {
this.addErrorMessage(NbBundle.getMessage(Chrome.class, "ExtractIE.getCookie.errMsg.errReadingIECookie", //NON-NLS
this.getName(), cookiesFile.getName()));
}
}
try {
blackboard.postArtifacts(bbartifacts, PARENT_MODULE_NAME);
blackboard.postArtifacts(bbartifacts, moduleName);
} catch (Blackboard.BlackboardException ex) {
logger.log(Level.SEVERE, "Error while trying to post Internet Explorer cookie artifact.", ex); //NON-NLS
this.addErrorMessage(Bundle.Extractor_errPostingArtifacts(getName()));
this.addErrorMessage(Bundle.ExtractIE_getCookie_errMsg_errPostinCookiess());
logger.log(Level.SEVERE, "Exception thrown while posting IE cookie artifact.", ex); //NON-NLS
}
}
/**
* Locates index.dat files, runs Pasco on them, and creates artifacts.
*/
@Messages({
"ExtractIE.getHistory.errMsg.errPostingHistory=Error posting Internet Explorer History artifacts."
})
private void getHistory() {
logger.log(Level.INFO, "Pasco results path: {0}", moduleTempResultsDir); //NON-NLS
boolean foundHistory = false;
//JIRA-2385: Why are we getting the pasco library path for datasource we process?
final File pascoRoot = InstalledFileLocator.getDefault().locate("pasco2", ExtractIE.class.getPackage().getName(), false); //NON-NLS
if (pascoRoot == null) {
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.unableToGetHist", this.getName()));
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.unableToGetHist", this.getName()));
logger.log(Level.SEVERE, "Error finding pasco program "); //NON-NLS
return;
}
@ -292,8 +309,8 @@ class ExtractIE extends Extract {
final String pascoHome = pascoRoot.getAbsolutePath();
logger.log(Level.INFO, "Pasco2 home: {0}", pascoHome); //NON-NLS
String pascoLibPath = pascoHome + File.separator + "pasco2.jar" + File.pathSeparator //NON-NLS
+ pascoHome + File.separator + "*";
PASCO_LIB_PATH = pascoHome + File.separator + "pasco2.jar" + File.pathSeparator //NON-NLS
+ pascoHome + File.separator + "*";
File resultsDir = new File(moduleTempResultsDir);
resultsDir.mkdirs();
@ -316,20 +333,18 @@ class ExtractIE extends Extract {
}
dataFound = true;
boolean foundHistory = false;
Collection<BlackboardArtifact> historyArtifacts = new ArrayList<>();
Collection<BlackboardArtifact> accountArtifacts = new ArrayList<>();
Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
String temps;
String indexFileName;
for (AbstractFile indexFile : indexFiles) {
/*
* Since each result represent an index.dat file, just create these
* files with the following notation: index<Number>.dat (i.e.
* index0.dat, index1.dat,..., indexN.dat), where <Number> is the
* obj_id of the file. Write each index.dat file to a temp
* directory. *
*/
String indexFileName = "index" + indexFile.getId() + ".dat"; //NON-NLS
String temps = RAImageIngestModule.getRATempPath(currentCase, "IE") + File.separator + indexFileName; //NON-NLS
// Since each result represent an index.dat file,
// just create these files with the following notation:
// index<Number>.dat (i.e. index0.dat, index1.dat,..., indexN.dat)
// Write each index.dat file to a temp directory.
//BlackboardArtifact bbart = fsc.newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY);
indexFileName = "index" + Integer.toString((int) indexFile.getId()) + ".dat"; //NON-NLS
//indexFileName = "index" + Long.toString(bbart.getArtifactID()) + ".dat";
temps = RAImageIngestModule.getRATempPath(currentCase, "IE") + File.separator + indexFileName; //NON-NLS
File datFile = new File(temps);
if (context.dataSourceIngestIsCancelled()) {
break;
@ -338,46 +353,42 @@ class ExtractIE extends Extract {
ContentUtils.writeToFile(indexFile, datFile, context::dataSourceIngestIsCancelled);
} catch (IOException e) {
logger.log(Level.WARNING, "Error while trying to write index.dat file " + datFile.getAbsolutePath(), e); //NON-NLS
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.errWriteFile", this.getName(),
datFile.getAbsolutePath()));
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.errWriteFile", this.getName(),
datFile.getAbsolutePath()));
continue;
}
String filename = "pasco2Result." + indexFile.getId() + ".txt"; //NON-NLS
boolean bPascProcSuccess = executePasco(pascoLibPath, temps, filename);
boolean bPascProcSuccess = executePasco(temps, filename);
if (context.dataSourceIngestIsCancelled()) {
return;
}
//At this point pasco2 proccessed the index file.
//Now fetch the results, parse them and the delete the file.
//At this point pasco2 proccessed the index files.
//Now fetch the results, parse them and the delete the files.
if (bPascProcSuccess) {
// Don't add TSK_OS_ACCOUNT artifacts to the ModuleDataEvent
HashMultimap<ARTIFACT_TYPE, BlackboardArtifact> artifacts = parsePascoOutput(indexFile, filename);
historyArtifacts.addAll(artifacts.get(TSK_WEB_HISTORY));
accountArtifacts.addAll(artifacts.get(TSK_OS_ACCOUNT));
bbartifacts.addAll(parsePascoOutput(indexFile, filename).stream()
.filter(bbart -> bbart.getArtifactTypeID() == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID())
.collect(Collectors.toList()));
foundHistory = true;
//Delete index<n>.dat file since it was succcessfully parsed by Pasco
//Delete index<n>.dat file since it was succcessfully by Pasco
datFile.delete();
} else {
logger.log(Level.WARNING, "pasco execution failed on: {0}", filename); //NON-NLS
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.errProcHist", this.getName()));
logger.log(Level.WARNING, "pasco execution failed on: {0}", this.getName()); //NON-NLS
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.errProcHist", this.getName()));
}
}
if (foundHistory) {
try {
blackboard.postArtifacts(historyArtifacts, PARENT_MODULE_NAME);
blackboard.postArtifacts(bbartifacts, moduleName);
} catch (Blackboard.BlackboardException ex) {
logger.log(Level.SEVERE, "Error while trying to post Internet Explorer history artifact.", ex); //NON-NLS
this.addErrorMessage(Bundle.Extractor_errPostingArtifacts(getName()));
}
try {
blackboard.postArtifacts(accountArtifacts, PARENT_MODULE_NAME);
} catch (Blackboard.BlackboardException ex) {
logger.log(Level.SEVERE, "Error while trying to post Internet Explorer os account artifact.", ex); //NON-NLS
this.addErrorMessage(Bundle.Extractor_errPostingArtifacts(getName()));
this.addErrorMessage(Bundle.ExtractIE_getHistory_errMsg_errPostingHistory());
logger.log(Level.SEVERE, "Exception thrown while posting IE cookie artifact.", ex); //NON-NLS
}
}
}
@ -388,22 +399,22 @@ class ExtractIE extends Extract {
* @param indexFilePath Path to local index.dat file to analyze
* @param outputFileName Name of file to save output to
*
* @return Success. False if there is an error executing pasco.
* @return false on error
*/
private boolean executePasco(String pascoLibraryPath, String indexFilePath, String outputFileName) {
private boolean executePasco(String indexFilePath, String outputFileName) {
boolean success = true;
try {
final String outputFileFullPath = moduleTempResultsDir + File.separator + outputFileName;
final String errFileFullPath = moduleTempResultsDir + File.separator + outputFileName + ".err"; //NON-NLS
logger.log(Level.INFO, "Writing pasco results to: {0}", outputFileFullPath); //NON-NLS
List<String> commandLine = Arrays.asList(
JAVA_PATH,
"-cp",//NON-NLS
pascoLibraryPath,
"isi.pasco2.Main", //NON-NLS
"-T", //NON-NLS
"history", //NON-NLS
indexFilePath);
List<String> commandLine = new ArrayList<>();
commandLine.add(JAVA_PATH);
commandLine.add("-cp"); //NON-NLS
commandLine.add(PASCO_LIB_PATH);
commandLine.add("isi.pasco2.Main"); //NON-NLS
commandLine.add("-T"); //NON-NLS
commandLine.add("history"); //NON-NLS
commandLine.add(indexFilePath);
ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
processBuilder.redirectOutput(new File(outputFileFullPath));
processBuilder.redirectError(new File(errFileFullPath));
@ -434,15 +445,16 @@ class ExtractIE extends Extract {
*
* @return A collection of created artifacts
*/
private HashMultimap<BlackboardArtifact.ARTIFACT_TYPE, BlackboardArtifact> parsePascoOutput(AbstractFile origFile, String pascoOutputFileName) {
HashMultimap<BlackboardArtifact.ARTIFACT_TYPE, BlackboardArtifact> bbartifacts = HashMultimap.create();
private Collection<BlackboardArtifact> parsePascoOutput(AbstractFile origFile, String pascoOutputFileName) {
Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
String fnAbs = moduleTempResultsDir + File.separator + pascoOutputFileName;
File file = new File(fnAbs);
if (file.exists() == false) {
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.notFound", this.getName(),
file.getName()));
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.notFound", this.getName(),
file.getName()));
logger.log(Level.WARNING, "Pasco Output not found: {0}", file.getPath()); //NON-NLS
return bbartifacts;
}
@ -452,151 +464,145 @@ class ExtractIE extends Extract {
if (file.length() == 0) {
return bbartifacts;
}
try (Scanner fileScanner = new Scanner(new FileInputStream(file.toString()));) {
// Keep a list of reported user accounts to avoid repeats.
// Initialize it with the empty string to represent an unknown user.
Set<String> reportedUserAccounts = Sets.newHashSet("");
while (fileScanner.hasNext()) {
parseLine(origFile, fileScanner.nextLine()).ifPresent(urlVisit -> {
Collection<BlackboardAttribute> bbattributes = Lists.newArrayList(new BlackboardAttribute(
TSK_URL, PARENT_MODULE_NAME,
urlVisit.url),
new BlackboardAttribute(
TSK_DATETIME_ACCESSED, PARENT_MODULE_NAME,
urlVisit.time),
//JIRA-2386: why are we adding an attribute that is always blank?
new BlackboardAttribute(
TSK_REFERRER, PARENT_MODULE_NAME,
""),
// @@@ NOte that other browser modules are adding TITLE in here for the title
new BlackboardAttribute(
TSK_PROG_NAME, PARENT_MODULE_NAME,
getName()),
new BlackboardAttribute(
TSK_USER_NAME, PARENT_MODULE_NAME,
urlVisit.user));
if (isIgnoredUrl(urlVisit.url) == false) {
bbattributes.add(new BlackboardAttribute(
TSK_DOMAIN, PARENT_MODULE_NAME,
urlVisit.domain));
}
try {
BlackboardArtifact bbart = origFile.newArtifact(TSK_WEB_HISTORY);
bbart.addAttributes(bbattributes);
bbartifacts.put(TSK_WEB_HISTORY, bbart);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error while trying to create Internet Explorer history artifact.", ex); //NON-NLS
addErrorMessage(
NbBundle.getMessage(Chrome.class, "ExtractIE.getHistory.errMsg.errProcHist", //NON-NLS
origFile.getName()));
}
if (reportedUserAccounts.contains(urlVisit.user) == false) {
try {
BlackboardArtifact osAttr = origFile.newArtifact(TSK_OS_ACCOUNT);
osAttr.addAttribute(new BlackboardAttribute(TSK_USER_NAME, PARENT_MODULE_NAME, urlVisit.user));
bbartifacts.put(TSK_OS_ACCOUNT, osAttr);
reportedUserAccounts.add(urlVisit.user);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error while trying to create Internet Explorer os account artifact.", ex); //NON-NLS
addErrorMessage(
NbBundle.getMessage(Chrome.class, "ExtractIE.getHistory.errMsg.errProcHist", //NON-NLS
origFile.getName()));
}
}
});
}
return bbartifacts;
Scanner fileScanner;
try {
fileScanner = new Scanner(new FileInputStream(file.toString()));
} catch (FileNotFoundException ex) {
addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.errParsing", this.getName(),
file.getName()));
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.errParsing", this.getName(),
file.getName()));
logger.log(Level.WARNING, "Unable to find the Pasco file at " + file.getPath(), ex); //NON-NLS
return bbartifacts;
}
while (fileScanner.hasNext()) {
String line = fileScanner.nextLine();
if (!line.startsWith("URL")) { //NON-NLS
continue;
}
}
String[] lineBuff = line.split("\\t"); //NON-NLS
private Optional<URLVisit> parseLine(AbstractFile origFile, String line) {
if (lineBuff.length < 4) {
logger.log(Level.INFO, "Found unrecognized IE history format."); //NON-NLS
continue;
}
if (!line.startsWith("URL")) { //NON-NLS
return Optional.empty();
}
String actime = lineBuff[3];
Long ftime = (long) 0;
String user = "";
String realurl = null;
String domain;
String[] lineBuff = line.split("\\t"); //NON-NLS
/*
* We've seen two types of lines: URL http://XYZ.com .... URL
* Visited: Joe@http://XYZ.com ....
*/
if (lineBuff[1].contains("@")) {
String url[] = lineBuff[1].split("@", 2);
if (lineBuff.length < 4) {
logger.log(Level.INFO, "Found unrecognized IE history format."); //NON-NLS
return Optional.empty();
}
String user;
String realurl;
/*
* Verify the left portion of the URL is valid.
*/
domain = extractDomain(url[0]);
if (domain != null && domain.isEmpty() == false) {
/*
* Use the entire input for the URL.
*/
realurl = lineBuff[1].trim();
} else {
/*
* Use the left portion of the input for the user, and the
* right portion for the host.
*/
user = url[0];
user = user.replace("Visited:", ""); //NON-NLS
user = user.replace(":Host:", ""); //NON-NLS
user = user.replaceAll("(:)(.*?)(:)", "");
user = user.trim();
realurl = url[1];
realurl = realurl.replace("Visited:", ""); //NON-NLS
realurl = realurl.replaceAll(":(.*?):", "");
realurl = realurl.replace(":Host:", ""); //NON-NLS
realurl = realurl.trim();
domain = extractDomain(realurl);
}
} else {
/*
* Use the entire input for the URL.
*/
realurl = lineBuff[1].trim();
domain = extractDomain(realurl);
}
/*
* We've seen two types of lines: URL http://XYZ.com .... URL Visited:
* Joe@http://XYZ.com ....
*/
if (lineBuff[1].contains("@")) {
String splitLine[] = lineBuff[1].split("@", 2);
//TODO: does the order matter? is this the same processing to both?
user = splitLine[0].replace("Visited:", "").replace(":Host:", "").replaceAll(":(.*?):", "").trim(); // NON-NLS
realurl = splitLine[1].replace("Visited:", "").replaceAll(":(.*?):", "").replace(":Host:", "").trim(); //NON-NLS
} else {
user = "";
realurl = lineBuff[1].trim();
}
if (!actime.isEmpty()) {
try {
Long epochtime = dateFormatter.parse(actime).getTime();
ftime = epochtime / 1000;
} catch (ParseException e) {
this.addErrorMessage(
NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.errParsingEntry",
this.getName()));
logger.log(Level.WARNING, String.format("Error parsing Pasco results, may have partial processing of corrupt file (id=%d)", origFile.getId()), e); //NON-NLS
}
}
String domain = NetworkUtils.extractDomain(realurl);
String actime = lineBuff[3];
Long ftime = (long) 0;
if (!actime.isEmpty()) {
try {
Long epochtime = new SimpleDateFormat(PASCO_DATE_FORMAT).parse(actime).getTime();
ftime = epochtime / 1000;
} catch (ParseException e) {
this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.errParsingEntry",
this.getName()));
logger.log(Level.WARNING, String.format("Error parsing Pasco results, may have partial processing of corrupt file (id=%d)", origFile.getId()), e); //NON-NLS
BlackboardArtifact bbart = origFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY);
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
RecentActivityExtracterModuleFactory.getModuleName(), realurl));
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "RecentActivity", EscapeUtil.decodeURL(realurl)));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
RecentActivityExtracterModuleFactory.getModuleName(), ftime));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
RecentActivityExtracterModuleFactory.getModuleName(), ""));
// @@@ NOte that other browser modules are adding TITLE in hre for the title
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"ExtractIE.moduleName.text")));
if (domain != null && domain.isEmpty() == false) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
RecentActivityExtracterModuleFactory.getModuleName(), domain));
}
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME,
RecentActivityExtracterModuleFactory.getModuleName(), user));
bbart.addAttributes(bbattributes);
// index the artifact for keyword search
this.indexArtifact(bbart);
bbartifacts.add(bbart);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error writing Internet Explorer web history artifact to the blackboard.", ex); //NON-NLS
}
}
return Optional.of(new URLVisit(user, realurl, domain, ftime));
fileScanner.close();
return bbartifacts;
}
/**
* Extract the domain from the supplied URL. This method does additional
* checks to detect invalid URLs.
*
* Determine if the URL should be ignored.
* @param url The URL from which to extract the domain.
*
* @param url The URL to test.
*
* @return True if the URL should be ignored; otherwise false.
* @return The domain.
*/
private boolean isIgnoredUrl(String url) {
/*
* Ignore blank URLs and URLs that begin with the matched text.
*/
return StringUtils.isBlank(url)
|| url.toLowerCase().startsWith(RESOURCE_URL_PREFIX);
}
@Immutable
private static class URLVisit {
private final String user;
private final String url;
private final String domain;
private final Long time;
URLVisit(String user, String url, String domain, Long ftime) {
this.user = user;
this.url = url;
this.domain = domain;
this.time = ftime;
private String extractDomain(String url) {
if (url == null || url.isEmpty()) {
return url;
}
if (url.toLowerCase().startsWith(RESOURCE_URL_PREFIX)) {
/*
* Ignore URLs that begin with the matched text.
*/
return null;
}
return NetworkUtils.extractDomain(url);
}
}

7
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractOs.java
View File

@ -28,6 +28,7 @@ import org.apache.commons.io.FilenameUtils;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
@ -39,7 +40,8 @@ import org.sleuthkit.datamodel.TskCoreException;
* Create OS INFO artifacts for the Operating Systems believed to be present on
* the data source.
*/
@Messages({"ExtractOs.parentModuleName=Recent Activity"})
@Messages({"ExtractOs.parentModuleName=Recent Activity",
"ExtractOS_progressMessage=Checking for OS"})
class ExtractOs extends Extract {
private static final Logger logger = Logger.getLogger(ExtractOs.class.getName());
@ -64,9 +66,10 @@ class ExtractOs extends Extract {
private Content dataSource;
@Override
void process(Content dataSource, IngestJobContext context) {
void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
this.dataSource = dataSource;
try {
progressBar.progress(Bundle.ExtractOS_progressMessage());
for (OS_TYPE value : OS_TYPE.values()) {
checkForOSFiles(value);
}

12
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java
View File

@ -52,6 +52,7 @@ import org.xml.sax.SAXException;
import java.nio.file.Path;
import static java.util.TimeZone.getTimeZone;
import org.openide.util.Lookup;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
@ -66,7 +67,8 @@ import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamExce
*/
@NbBundle.Messages({
"RegRipperNotFound=Autopsy RegRipper executable not found.",
"RegRipperFullNotFound=Full version RegRipper executable not found."
"RegRipperFullNotFound=Full version RegRipper executable not found.",
"Progress_Message_Analyze_Registry=Analyzing Registry Files"
})
class ExtractRegistry extends Extract {
@ -415,7 +417,7 @@ class ExtractRegistry extends Extract {
Element artroot = (Element) artroots.item(0);
NodeList myartlist = artroot.getChildNodes();
String parentModuleName = NbBundle.getMessage(this.getClass(), "ExtractRegistry.parentModuleName.noSpace");
String parentModuleName = RecentActivityExtracterModuleFactory.getModuleName();
String winver = "";
// If all artifact nodes should really go under one Blackboard artifact, need to process it differently
@ -829,7 +831,7 @@ class ExtractRegistry extends Extract {
*/
private boolean parseSamPluginOutput(String regFilePath, AbstractFile regAbstractFile) {
File regfile = new File(regFilePath);
String parentModuleName = NbBundle.getMessage(this.getClass(), "ExtractRegistry.parentModuleName.noSpace");
String parentModuleName = RecentActivityExtracterModuleFactory.getModuleName();
SimpleDateFormat regRipperTimeFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy 'Z'");
regRipperTimeFormat.setTimeZone(getTimeZone("GMT"));
try (BufferedReader bufferedReader = new BufferedReader(new FileReader(regfile))) {
@ -964,9 +966,11 @@ class ExtractRegistry extends Extract {
}
@Override
public void process(Content dataSource, IngestJobContext context) {
public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
this.dataSource = dataSource;
this.context = context;
progressBar.progress(Bundle.Progress_Message_Analyze_Registry());
analyzeRegistryFiles();
}

642
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java Executable file
View File

@ -0,0 +1,642 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.recentactivity;
import com.dd.plist.NSArray;
import com.dd.plist.NSDate;
import com.dd.plist.NSDictionary;
import com.dd.plist.NSObject;
import com.dd.plist.NSString;
import com.dd.plist.PropertyListFormatException;
import com.dd.plist.PropertyListParser;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Level;
import javax.xml.parsers.ParserConfigurationException;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.autopsy.recentactivity.BinaryCookieReader.Cookie;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
import org.xml.sax.SAXException;
/**
* Extract the bookmarks, cookies, downloads and history from Safari
*
*/
final class ExtractSafari extends Extract {
private final IngestServices services = IngestServices.getInstance();
// visit_time uses an epoch of Jan 1, 2001 thus the addition of 978307200
private static final String HISTORY_QUERY = "SELECT url, title, visit_time + 978307200 as time FROM 'history_items' JOIN history_visits ON history_item = history_items.id;"; //NON-NLS
private static final String HISTORY_FILE_NAME = "History.db"; //NON-NLS
private static final String BOOKMARK_FILE_NAME = "Bookmarks.plist"; //NON-NLS
private static final String DOWNLOAD_FILE_NAME = "Downloads.plist"; //NON-NLS
private static final String COOKIE_FILE_NAME = "Cookies.binarycookies"; //NON-NLS
private static final String COOKIE_FOLDER = "Cookies";
private static final String SAFARI_FOLDER = "Safari";
private static final String HEAD_URL = "url"; //NON-NLS
private static final String HEAD_TITLE = "title"; //NON-NLS
private static final String HEAD_TIME = "time"; //NON-NLS
private static final String PLIST_KEY_CHILDREN = "Children"; //NON-NLS
private static final String PLIST_KEY_URL = "URLString"; //NON-NLS
private static final String PLIST_KEY_URI = "URIDictionary"; //NON-NLS
private static final String PLIST_KEY_TITLE = "title"; //NON-NLS
private static final String PLIST_KEY_DOWNLOAD_URL = "DownloadEntryURL"; //NON-NLS
private static final String PLIST_KEY_DOWNLOAD_DATE = "DownloadEntryDateAddedKey"; //NON-NLS
private static final String PLIST_KEY_DOWNLOAD_PATH = "DownloadEntryPath"; //NON-NLS
private static final String PLIST_KEY_DOWNLOAD_HISTORY = "DownloadHistory"; //NON-NLS
private static final Logger LOG = Logger.getLogger(ExtractSafari.class.getName());
@Messages({
"ExtractSafari_Module_Name=Safari",
"ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files.",
"ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files",
"ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files",
"Progress_Message_Safari_History=Safari History",
"Progress_Message_Safari_Bookmarks=Safari Bookmarks",
"Progress_Message_Safari_Cookies=Safari Cookies",
"Progress_Message_Safari_Downloads=Safari Downloads",
})
/**
* Extract the bookmarks, cookies, downloads and history from Safari.
*
*/
ExtractSafari() {
}
@Override
protected String getName() {
return Bundle.ExtractSafari_Module_Name();
}
@Override
void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
setFoundData(false);
progressBar.progress(Bundle.Progress_Message_Safari_Cookies());
try {
processHistoryDB(dataSource, context);
} catch (IOException | TskCoreException ex) {
this.addErrorMessage(Bundle.ExtractSafari_Error_Getting_History());
LOG.log(Level.SEVERE, "Exception thrown while processing history file: {0}", ex); //NON-NLS
}
progressBar.progress(Bundle.Progress_Message_Safari_Bookmarks());
try {
processBookmarkPList(dataSource, context);
} catch (IOException | TskCoreException | SAXException | PropertyListFormatException | ParseException | ParserConfigurationException ex) {
this.addErrorMessage(Bundle.ExtractSafari_Error_Parsing_Bookmark());
LOG.log(Level.SEVERE, "Exception thrown while parsing Safari Bookmarks file: {0}", ex); //NON-NLS
}
progressBar.progress(Bundle.Progress_Message_Safari_Downloads());
try {
processDownloadsPList(dataSource, context);
} catch (IOException | TskCoreException | SAXException | PropertyListFormatException | ParseException | ParserConfigurationException ex) {
this.addErrorMessage(Bundle.ExtractSafari_Error_Parsing_Bookmark());
LOG.log(Level.SEVERE, "Exception thrown while parsing Safari Download.plist file: {0}", ex); //NON-NLS
}
progressBar.progress(Bundle.Progress_Message_Safari_Cookies());
try {
processBinaryCookieFile(dataSource, context);
} catch (IOException | TskCoreException ex) {
this.addErrorMessage(Bundle.ExtractSafari_Error_Parsing_Cookies());
LOG.log(Level.SEVERE, "Exception thrown while processing Safari cookies file: {0}", ex); //NON-NLS
}
}
/**
* Finds the all of the history.db files in the case looping through them to
* find all of the history artifacts.
*
* @throws TskCoreException
* @throws IOException
*/
private void processHistoryDB(Content dataSource, IngestJobContext context) throws TskCoreException, IOException {
FileManager fileManager = getCurrentCase().getServices().getFileManager();
List<AbstractFile> historyFiles = fileManager.findFiles(dataSource, HISTORY_FILE_NAME, SAFARI_FOLDER);
if (historyFiles == null || historyFiles.isEmpty()) {
return;
}
setFoundData(true);
for (AbstractFile historyFile : historyFiles) {
if (context.dataSourceIngestIsCancelled()) {
break;
}
getHistory(context, historyFile);
}
}
/**
* Finds all Bookmark.plist files and looks for bookmark entries.
* @param dataSource
* @param context
* @throws TskCoreException
* @throws IOException
* @throws SAXException
* @throws PropertyListFormatException
* @throws ParseException
* @throws ParserConfigurationException
*/
private void processBookmarkPList(Content dataSource, IngestJobContext context) throws TskCoreException, IOException, SAXException, PropertyListFormatException, ParseException, ParserConfigurationException {
FileManager fileManager = getCurrentCase().getServices().getFileManager();
List<AbstractFile> files = fileManager.findFiles(dataSource, BOOKMARK_FILE_NAME, SAFARI_FOLDER);
if (files == null || files.isEmpty()) {
return;
}
setFoundData(true);
for (AbstractFile file : files) {
if (context.dataSourceIngestIsCancelled()) {
break;
}
getBookmarks(context, file);
}
}
/**
* Process the safari download.plist file.
*
* @param dataSource
* @param context
* @throws TskCoreException
* @throws IOException
* @throws SAXException
* @throws PropertyListFormatException
* @throws ParseException
* @throws ParserConfigurationException
*/
private void processDownloadsPList(Content dataSource, IngestJobContext context) throws TskCoreException, IOException, SAXException, PropertyListFormatException, ParseException, ParserConfigurationException {
FileManager fileManager = getCurrentCase().getServices().getFileManager();
List<AbstractFile> files = fileManager.findFiles(dataSource, DOWNLOAD_FILE_NAME, SAFARI_FOLDER);
if (files == null || files.isEmpty()) {
return;
}
setFoundData(true);
for (AbstractFile file : files) {
if (context.dataSourceIngestIsCancelled()) {
break;
}
getDownloads(dataSource, context, file);
}
}
/**
* Process the Safari Cookie file.
* @param dataSource
* @param context
* @throws TskCoreException
* @throws IOException
*/
private void processBinaryCookieFile(Content dataSource, IngestJobContext context) throws TskCoreException, IOException {
FileManager fileManager = getCurrentCase().getServices().getFileManager();
List<AbstractFile> files = fileManager.findFiles(dataSource, COOKIE_FILE_NAME, COOKIE_FOLDER);
if (files == null || files.isEmpty()) {
return;
}
setFoundData(true);
for (AbstractFile file : files) {
if (context.dataSourceIngestIsCancelled()) {
break;
}
getCookies(context, file);
}
}
/**
* Creates a temporary copy of historyFile and creates a list of
* BlackboardArtifacts for the history information in the file.
*
* @param historyFile AbstractFile version of the history file from the case
* @throws TskCoreException
* @throws IOException
*/
private void getHistory(IngestJobContext context, AbstractFile historyFile) throws TskCoreException, IOException {
if (historyFile.getSize() == 0) {
return;
}
File tempHistoryFile = createTemporaryFile(context, historyFile);
try {
ContentUtils.writeToFile(historyFile, tempHistoryFile, context::dataSourceIngestIsCancelled);
} catch (IOException ex) {
throw new IOException("Error writingToFile: " + historyFile, ex); //NON-NLS
}
try {
Collection<BlackboardArtifact> bbartifacts = getHistoryArtifacts(historyFile, tempHistoryFile.toPath());
if (!bbartifacts.isEmpty()) {
services.fireModuleDataEvent(new ModuleDataEvent(
RecentActivityExtracterModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
}
} finally {
tempHistoryFile.delete();
}
}
/**
* Creates a temporary bookmark file from the AbstractFile and creates
* BlackboardArtifacts for the any bookmarks found.
*
* @param context IngestJobContext object
* @param file AbstractFile from case
* @throws TskCoreException
* @throws IOException
* @throws SAXException
* @throws PropertyListFormatException
* @throws ParseException
* @throws ParserConfigurationException
*/
private void getBookmarks(IngestJobContext context, AbstractFile file) throws TskCoreException, IOException, SAXException, PropertyListFormatException, ParseException, ParserConfigurationException {
if (file.getSize() == 0) {
return;
}
File tempFile = createTemporaryFile(context, file);
try {
Collection<BlackboardArtifact> bbartifacts = getBookmarkArtifacts(file, tempFile);
if (!bbartifacts.isEmpty()) {
services.fireModuleDataEvent(new ModuleDataEvent(
RecentActivityExtracterModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK, bbartifacts));
}
} finally {
tempFile.delete();
}
}
/**
* Creates a temporary downloads file from the AbstractFile and creates
* BlackboardArtifacts for the any downloads found.
*
* @param context IngestJobContext object
* @param file AbstractFile from case
* @throws TskCoreException
* @throws IOException
* @throws SAXException
* @throws PropertyListFormatException
* @throws ParseException
* @throws ParserConfigurationException
*/
private void getDownloads(Content dataSource, IngestJobContext context, AbstractFile file) throws TskCoreException, IOException, SAXException, PropertyListFormatException, ParseException, ParserConfigurationException {
if (file.getSize() == 0) {
return;
}
File tempFile = createTemporaryFile(context, file);
try {
Collection<BlackboardArtifact> bbartifacts = getDownloadArtifacts(dataSource, file, tempFile);
if (!bbartifacts.isEmpty()) {
services.fireModuleDataEvent(new ModuleDataEvent(
RecentActivityExtracterModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD, bbartifacts));
}
} finally {
if (tempFile != null) {
tempFile.delete();
}
}
}
/**
* Creates a temporary copy of the Cookie file and creates a list of cookie
* BlackboardArtifacts.
*
* @param context IngetstJobContext
* @param file Original Cookie file from the case
* @throws TskCoreException
* @throws IOException
*/
private void getCookies(IngestJobContext context, AbstractFile file) throws TskCoreException, IOException {
if (file.getSize() == 0) {
return;
}
File tempFile = null;
try {
tempFile = createTemporaryFile(context, file);
Collection<BlackboardArtifact> bbartifacts = getCookieArtifacts(file, tempFile);
if (!bbartifacts.isEmpty()) {
services.fireModuleDataEvent(new ModuleDataEvent(
RecentActivityExtracterModuleFactory.getModuleName(),
BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE, bbartifacts));
}
} finally {
if (tempFile != null) {
tempFile.delete();
}
}
}
/**
* Queries the history db for the history information creating a list of
* BlackBoardArtifact for each row returned from the db.
*
* @param origFile AbstractFile of the history file from the case
* @param tempFilePath Path to temporary copy of the history db
* @return Blackboard Artifacts for the history db or null if there are no
* history artifacts
* @throws TskCoreException
*/
private Collection<BlackboardArtifact> getHistoryArtifacts(AbstractFile origFile, Path tempFilePath) throws TskCoreException {
List<HashMap<String, Object>> historyList = this.dbConnect(tempFilePath.toString(), HISTORY_QUERY);
if (historyList == null || historyList.isEmpty()) {
return null;
}
Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
for (HashMap<String, Object> row : historyList) {
String url = row.get(HEAD_URL).toString();
String title = row.get(HEAD_TITLE).toString();
Long time = (Double.valueOf(row.get(HEAD_TIME).toString())).longValue();
BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY);
bbart.addAttributes(createHistoryAttribute(url, time, null, title,
this.getName(), NetworkUtils.extractDomain(url), null));
bbartifacts.add(bbart);
}
return bbartifacts;
}
/**
* Parses the temporary version of bookmarks.plist and creates
*
* @param origFile The origFile Bookmark.plist file from the case
* @param tempFile The temporary local version of Bookmark.plist
* @return Collection of BlackboardArtifacts for the bookmarks in origFile
* @throws IOException
* @throws PropertyListFormatException
* @throws ParseException
* @throws ParserConfigurationException
* @throws SAXException
* @throws TskCoreException
*/
private Collection<BlackboardArtifact> getBookmarkArtifacts(AbstractFile origFile, File tempFile) throws IOException, PropertyListFormatException, ParseException, ParserConfigurationException, SAXException, TskCoreException {
Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
try {
NSDictionary root = (NSDictionary) PropertyListParser.parse(tempFile);
parseBookmarkDictionary(bbartifacts, origFile, root);
} catch (PropertyListFormatException ex) {
PropertyListFormatException plfe = new PropertyListFormatException(origFile.getName() + ": " + ex.getMessage());
plfe.setStackTrace(ex.getStackTrace());
throw plfe;
} catch (ParseException ex) {
ParseException pe = new ParseException(origFile.getName() + ": " + ex.getMessage(), ex.getErrorOffset());
pe.setStackTrace(ex.getStackTrace());
throw pe;
} catch (ParserConfigurationException ex) {
ParserConfigurationException pce = new ParserConfigurationException(origFile.getName() + ": " + ex.getMessage());
pce.setStackTrace(ex.getStackTrace());
throw pce;
} catch (SAXException ex) {
SAXException se = new SAXException(origFile.getName() + ": " + ex.getMessage());
se.setStackTrace(ex.getStackTrace());
throw se;
}
return bbartifacts;
}
/**
* Finds the download entries in the tempFile and creates a list of artifacts from them.
*
* @param origFile Download.plist file from case
* @param tempFile Temporary copy of download.plist file
* @return Collection of BlackboardArtifacts for the downloads in origFile
* @throws IOException
* @throws PropertyListFormatException
* @throws ParseException
* @throws ParserConfigurationException
* @throws SAXException
* @throws TskCoreException
*/
private Collection<BlackboardArtifact> getDownloadArtifacts(Content dataSource, AbstractFile origFile, File tempFile)throws IOException, PropertyListFormatException, ParseException, ParserConfigurationException, SAXException, TskCoreException {
Collection<BlackboardArtifact> bbartifacts = null;
try {
while(true){
NSDictionary root = (NSDictionary)PropertyListParser.parse(tempFile);
if(root == null)
break;
NSArray nsArray = (NSArray)root.get(PLIST_KEY_DOWNLOAD_HISTORY);
if(nsArray == null)
break;
NSObject[] objectArray = nsArray.getArray();
bbartifacts = new ArrayList<>();
for(NSObject obj: objectArray){
if(obj instanceof NSDictionary){
bbartifacts.add(parseDownloadDictionary(dataSource, origFile, (NSDictionary)obj));
}
}
break;
}
} catch (PropertyListFormatException ex) {
PropertyListFormatException plfe = new PropertyListFormatException(origFile.getName() + ": " + ex.getMessage());
plfe.setStackTrace(ex.getStackTrace());
throw plfe;
} catch (ParseException ex) {
ParseException pe = new ParseException(origFile.getName() + ": " + ex.getMessage(), ex.getErrorOffset());
pe.setStackTrace(ex.getStackTrace());
throw pe;
} catch (ParserConfigurationException ex) {
ParserConfigurationException pce = new ParserConfigurationException(origFile.getName() + ": " + ex.getMessage());
pce.setStackTrace(ex.getStackTrace());
throw pce;
} catch (SAXException ex) {
SAXException se = new SAXException(origFile.getName() + ": " + ex.getMessage());
se.setStackTrace(ex.getStackTrace());
throw se;
}
return bbartifacts;
}
/**
* Finds the cookies in the tempFile creating a list of BlackboardArtifacts
* each representing one cookie.
*
* @param origFile Original Cookies.binarycookie file from case
* @param tempFile Temporary copy of the cookies file
* @return List of Blackboard Artifacts, one for each cookie
* @throws TskCoreException
* @throws IOException
*/
private Collection<BlackboardArtifact> getCookieArtifacts(AbstractFile origFile, File tempFile) throws TskCoreException, IOException {
Collection<BlackboardArtifact> bbartifacts = null;
BinaryCookieReader reader = BinaryCookieReader.initalizeReader(tempFile);
if (reader != null) {
bbartifacts = new ArrayList<>();
Iterator<Cookie> iter = reader.iterator();
while (iter.hasNext()) {
Cookie cookie = iter.next();
BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE);
bbart.addAttributes(createCookieAttributes(cookie.getURL(), cookie.getCreationDate(), cookie.getName(), cookie.getValue(), this.getName(), NetworkUtils.extractDomain(cookie.getURL())));
bbartifacts.add(bbart);
}
}
return bbartifacts;
}
/**
* Parses the plist object to find the bookmark child objects, then creates
* an artifact with the bookmark information.
*
* @param bbartifacts BlackboardArtifact list to add new the artifacts to
* @param origFile The origFile Bookmark.plist file from the case
* @param root NSDictionary object to parse
* @throws TskCoreException
*/
private void parseBookmarkDictionary(Collection<BlackboardArtifact> bbartifacts, AbstractFile origFile, NSDictionary root) throws TskCoreException {
if (root.containsKey(PLIST_KEY_CHILDREN)) {
NSArray children = (NSArray) root.objectForKey(PLIST_KEY_CHILDREN);
if (children != null) {
for (NSObject obj : children.getArray()) {
parseBookmarkDictionary(bbartifacts, origFile, (NSDictionary) obj);
}
}
} else if (root.containsKey(PLIST_KEY_URL)) {
String url = null;
String title = null;
NSString nsstr = (NSString) root.objectForKey(PLIST_KEY_URL);
if (nsstr != null) {
url = nsstr.toString();
}
NSDictionary dic = (NSDictionary) root.get(PLIST_KEY_URI);
nsstr = (NSString) root.objectForKey(PLIST_KEY_TITLE);
if (nsstr != null) {
title = ((NSString) dic.get(PLIST_KEY_TITLE)).toString();
}
if (url != null || title != null) {
BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
bbart.addAttributes(createBookmarkAttributes(url, title, null, getName(), NetworkUtils.extractDomain(url)));
bbartifacts.add(bbart);
}
}
}
/**
* Parse the NSDictionary object that represents one download.
*
* @param origFile Download.plist file from the case
* @param entry One NSDictionary Object that represents one download
* instance
* @return a Blackboard Artifact for the download.
* @throws TskCoreException
*/
private BlackboardArtifact parseDownloadDictionary(Content dataSource, AbstractFile origFile, NSDictionary entry) throws TskCoreException {
String url = null;
String path = null;
Long time = null;
Long pathID = null;
NSString nsstring = (NSString) entry.get(PLIST_KEY_DOWNLOAD_URL);
if (nsstring != null) {
url = nsstring.toString();
}
nsstring = (NSString) entry.get(PLIST_KEY_DOWNLOAD_PATH);
if (nsstring != null) {
path = nsstring.toString();
pathID = Util.findID(dataSource, path);
}
NSDate date = (NSDate) entry.get(PLIST_KEY_DOWNLOAD_DATE);
if (date != null) {
time = date.getDate().getTime();
}
BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD);
bbart.addAttributes(this.createDownloadAttributes(path, pathID, url, time, NetworkUtils.extractDomain(url), getName()));
return bbart;
}
}

174
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java
View File

@ -44,12 +44,14 @@ import java.util.Set;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
@ -63,6 +65,15 @@ import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ReadContentInputStream.ReadContentInputStreamException;
import org.sleuthkit.datamodel.TskCoreException;
@Messages({
"Progress_Message_Firefox_History=Firefox History",
"Progress_Message_Firefox_Bookmarks=Firefox Bookmarks",
"Progress_Message_Firefox_Cookies=Firefox Cookies",
"Progress_Message_Firefox_Downloads=Firefox Downloads",
"Progress_Message_Firefox_FormHistory=Firefox Form History",
"Progress_Message_Firefox_AutoFill=Firefox Auto Fill"
})
/**
* Firefox recent activity extraction
*/
@ -96,15 +107,27 @@ class Firefox extends Extract {
}
@Override
public void process(Content dataSource, IngestJobContext context) {
public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
this.dataSource = dataSource;
this.context = context;
dataFound = false;
progressBar.progress(Bundle.Progress_Message_Firefox_History());
this.getHistory();
progressBar.progress(Bundle.Progress_Message_Firefox_Bookmarks());
this.getBookmark();
progressBar.progress(Bundle.Progress_Message_Firefox_Downloads());
this.getDownload();
progressBar.progress(Bundle.Progress_Message_Firefox_Cookies());
this.getCookie();
progressBar.progress(Bundle.Progress_Message_Firefox_FormHistory());
this.getFormsHistory();
progressBar.progress(Bundle.Progress_Message_Firefox_AutoFill());
this.getAutofillProfiles();
}
@ -165,31 +188,25 @@ class Firefox extends Extract {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
((url != null) ? url : ""))); //NON-NLS
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "RecentActivity", ((result.get("url").toString() != null) ? EscapeUtil.decodeURL(result.get("url").toString()) : "")));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("visit_date").toString())))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("ref").toString() != null) ? result.get("ref").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("title").toString() != null) ? result.get("title").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "Firefox.moduleName")));
String domain = extractDomain(url);
if (domain != null && domain.isEmpty() == false) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"), domain)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(), domain)); //NON-NLS
}
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY, historyFile, bbattributes);
@ -261,29 +278,23 @@ class Firefox extends Extract {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
((url != null) ? url : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("title").toString() != null) ? result.get("title").toString() : ""))); //NON-NLS
if (Long.valueOf(result.get("dateAdded").toString()) > 0) { //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("dateAdded").toString())))); //NON-NLS
}
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "Firefox.moduleName")));
String domain = extractDomain(url);
if (domain != null && domain.isEmpty() == false) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
domain)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(), domain)); //NON-NLS
}
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK, bookmarkFile, bbattributes);
@ -363,38 +374,31 @@ class Firefox extends Extract {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
((host != null) ? host : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("lastAccessed").toString())))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("name").toString() != null) ? result.get("name").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("value").toString() != null) ? result.get("value").toString() : ""))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "Firefox.moduleName")));
if (checkColumn == true) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
(Long.valueOf(result.get("creationTime").toString())))); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("creationTime").toString())))); //NON-NLS
}
String domain = extractDomain(host);
if (domain != null && domain.isEmpty() == false) {
domain = domain.replaceFirst("^\\.+(?!$)", "");
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"), domain));
RecentActivityExtracterModuleFactory.getModuleName(), domain));
}
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_COOKIE, cookiesFile, bbattributes);
@ -479,13 +483,11 @@ class Firefox extends Extract {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
source)); //NON-NLS
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "RecentActivity", ((result.get("source").toString() != null) ? EscapeUtil.decodeURL(result.get("source").toString()) : "")));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("startTime").toString())))); //NON-NLS
String target = result.get("target").toString(); //NON-NLS
@ -494,14 +496,12 @@ class Firefox extends Extract {
try {
String decodedTarget = URLDecoder.decode(target.replaceAll("file:///", ""), "UTF-8"); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
decodedTarget));
long pathID = Util.findID(dataSource, decodedTarget);
if (pathID != -1) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH_ID,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
pathID));
}
} catch (UnsupportedEncodingException ex) {
@ -511,14 +511,12 @@ class Firefox extends Extract {
}
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "Firefox.moduleName")));
String domain = extractDomain(source);
if (domain != null && domain.isEmpty() == false) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
domain)); //NON-NLS
}
@ -603,8 +601,7 @@ class Firefox extends Extract {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
url)); //NON-NLS
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "RecentActivity", ((result.get("source").toString() != null) ? EscapeUtil.decodeURL(result.get("source").toString()) : "")));
//TODO Revisit usage of deprecated constructor as per TSK-583
@ -615,14 +612,12 @@ class Firefox extends Extract {
try {
String decodedTarget = URLDecoder.decode(target.replaceAll("file:///", ""), "UTF-8"); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
decodedTarget));
long pathID = Util.findID(dataSource, decodedTarget);
if (pathID != -1) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH_ID,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
pathID));
}
} catch (UnsupportedEncodingException ex) {
@ -631,19 +626,15 @@ class Firefox extends Extract {
}
}
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
Long.valueOf(result.get("lastModified").toString()))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
RecentActivityExtracterModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(), "Firefox.moduleName")));
String domain = extractDomain(url);
if (domain != null && domain.isEmpty() == false) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
NbBundle.getMessage(this.getClass(),
"Firefox.parentModuleName.noSpace"),
domain)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(), domain)); //NON-NLS
}
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_DOWNLOAD, downloadsFile, bbattributes);
@ -740,27 +731,26 @@ class Firefox extends Extract {
}
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
fieldName)); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
RecentActivityExtracterModuleFactory.getModuleName(),
((result.get("value").toString() != null) ? result.get("value").toString() : ""))); //NON-NLS
// Newer versions of firefox have additional columns
if (isFirefoxV64) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
(Long.valueOf(result.get("firstUsed").toString()) / 1000000))); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("firstUsed").toString()) / 1000000))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
(Long.valueOf(result.get("lastUsed").toString()) / 1000000))); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
(Long.valueOf(result.get("lastUsed").toString()) / 1000000))); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
(Integer.valueOf(result.get("timesUsed").toString())))); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
(Integer.valueOf(result.get("timesUsed").toString())))); //NON-NLS
}
// Add artifact
BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_FORM_AUTOFILL, formHistoryFile, bbattributes);
@ -901,32 +891,34 @@ class Firefox extends Extract {
try {
Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME_PERSON,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
name)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
name)); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
email)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
email)); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
phoneNumber)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
phoneNumber)); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LOCATION,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
mailingAddress)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
mailingAddress)); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
datetimeCreated)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
datetimeCreated)); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
datetimeLastUsed)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
datetimeLastUsed)); //NON-NLS
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT,
NbBundle.getMessage(this.getClass(), "Firefox.parentModuleName"),
timesUsed)); //NON-NLS
RecentActivityExtracterModuleFactory.getModuleName(),
timesUsed)); //NON-NLS
BlackboardArtifact bbart = profileFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_FORM_ADDRESS);

21
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java
View File

@ -23,6 +23,7 @@
package org.sleuthkit.autopsy.recentactivity;
import java.io.File;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
@ -59,8 +60,10 @@ public final class RAImageIngestModule implements DataSourceIngestModule {
this.context = context;
Extract iexplore;
Extract edge;
try {
iexplore = new ExtractIE();
edge = new ExtractEdge();
} catch (NoCurrentCaseException ex) {
throw new IngestModuleException(ex.getMessage(), ex);
}
@ -72,10 +75,13 @@ public final class RAImageIngestModule implements DataSourceIngestModule {
Extract SEUQA = new SearchEngineURLQueryAnalyzer();
Extract osExtract = new ExtractOs();
Extract dataSourceAnalyzer = new DataSourceUsageAnalyzer();
Extract safari = new ExtractSafari();
extractors.add(chrome);
extractors.add(firefox);
extractors.add(iexplore);
extractors.add(edge);
extractors.add(safari);
extractors.add(recentDocuments);
extractors.add(SEUQA); // this needs to run after the web browser modules
extractors.add(registry); // this should run after quicker modules like the browser modules and needs to run before the DataSourceUsageAnalyzer
@ -85,6 +91,8 @@ public final class RAImageIngestModule implements DataSourceIngestModule {
browserExtractors.add(chrome);
browserExtractors.add(firefox);
browserExtractors.add(iexplore);
browserExtractors.add(edge);
browserExtractors.add(safari);
for (Extract extractor : extractors) {
extractor.init();
@ -112,7 +120,7 @@ public final class RAImageIngestModule implements DataSourceIngestModule {
progressBar.progress(extracter.getName(), i);
try {
extracter.process(dataSource, context);
extracter.process(dataSource, context, progressBar);
} catch (Exception ex) {
logger.log(Level.SEVERE, "Exception occurred in " + extracter.getName(), ex); //NON-NLS
subCompleted.append(NbBundle.getMessage(this.getClass(), "RAImageIngestModule.process.errModFailed",
@ -227,4 +235,15 @@ public final class RAImageIngestModule implements DataSourceIngestModule {
}
return tmpDir;
}
/**
* Get relative path for module output folder.
*
* @throws NoCurrentCaseException if there is no open case.
* @return the relative path of the module output folder
*/
static String getRelModuleOutputPath() throws NoCurrentCaseException {
return Paths.get(Case.getCurrentCaseThrows().getModuleOutputDirectoryRelativePath(),
"RecentActivity").normalize().toString() ; //NON-NLS
}
}

9
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java
View File

@ -29,6 +29,7 @@ import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import java.util.Collection;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.coreutils.JLNK;
import org.sleuthkit.autopsy.coreutils.JLnkParser;
import org.sleuthkit.autopsy.coreutils.JLnkParserException;
@ -53,6 +54,10 @@ class RecentDocumentsByLnk extends Extract {
private IngestServices services = IngestServices.getInstance();
private Content dataSource;
private IngestJobContext context;
@Messages({
"Progress_Message_Extract_Resent_Docs=Recent Documents",
})
/**
* Find the documents that Windows stores about recent documents and make
@ -123,10 +128,12 @@ class RecentDocumentsByLnk extends Extract {
}
@Override
public void process(Content dataSource, IngestJobContext context) {
public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
this.dataSource = dataSource;
this.context = context;
dataFound = false;
progressBar.progress(Bundle.Progress_Message_Extract_Resent_Docs());
this.getRecentDocuments();
}
}

8
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java
View File

@ -33,6 +33,7 @@ import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.coreutils.XMLUtil;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
import org.sleuthkit.autopsy.ingest.IngestServices;
@ -62,7 +63,8 @@ import org.xml.sax.SAXException;
"cannotBuildXmlParser=Unable to build XML parser: ",
"cannotLoadSEUQA=Unable to load Search Engine URL Query Analyzer settings file, SEUQAMappings.xml: ",
"cannotParseXml=Unable to parse XML file: ",
"# {0} - file name", "SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}."
"# {0} - file name", "SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}.",
"Progress_Message_Find_Search_Query=Find Search Queries"
})
class SearchEngineURLQueryAnalyzer extends Extract {
@ -394,9 +396,11 @@ class SearchEngineURLQueryAnalyzer extends Extract {
}
@Override
public void process(Content dataSource, IngestJobContext context) {
public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
this.dataSource = dataSource;
this.context = context;
progressBar.progress(Bundle.Progress_Message_Find_Search_Query());
this.findSearchQueries();
logger.log(Level.INFO, "Search Engine stats: \n{0}", getTotals()); //NON-NLS
}

2
branding/core/core.jar/org/netbeans/core/startup/Bundle.properties
View File

@ -1,5 +1,5 @@
#Updated by build script
#Fri, 01 Mar 2019 12:26:15 +0100
#Wed, 06 Mar 2019 16:04:26 +0100
LBL_splash_window_title=Starting Autopsy
SPLASH_HEIGHT=314
SPLASH_WIDTH=538

2
branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties
View File

@ -1,4 +1,4 @@
#Updated by build script
#Fri, 01 Mar 2019 12:26:15 +0100
#Wed, 06 Mar 2019 16:04:26 +0100
CTL_MainWindow_Title=Autopsy 4.10.0
CTL_MainWindow_Title_No_Project=Autopsy 4.10.0

33
thirdparty/ESEDatabaseView/ESEDatabaseView.cfg vendored Executable file
View File

@ -0,0 +1,33 @@
[General]
AutoSizeColumnsOnTableChange=1
MarkOddEvenRows=0
ShowGridLines=0
SaveFilterIndex=0
ShowInfoTip=1
AutoDetectDateTime=1
ConvertGMTToLocalTime=0
BinaryURLAsString=0
DetectBinaryUTF16=0
DetectBinaryAscii=0
SaveFileEncoeding=0
AlignNumbersToRight=0
UseQuickFilter=0
QuickFilterString=
QuickFilterColumnsMode=1
QuickFilterFindMode=1
QuickFilterShowHide=1
TrayIcon=0
WinPos=2C 00 00 00 00 00 00 00 01 00 00 00 FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF 22 02 00 00 9D FC FF FF 2A 07 00 00 4A FF FF FF
Columns=
Sort=0
[RecentFiles]
0=
1=
2=
3=
4=
5=
6=
7=
8=
9=

BIN
thirdparty/ESEDatabaseView/ESEDatabaseView.chm vendored Executable file
View File

Binary file not shown.

BIN
thirdparty/ESEDatabaseView/ESEDatabaseView.exe vendored Executable file
View File

Binary file not shown.

392
thirdparty/ESEDatabaseView/readme.txt vendored Executable file
View File

@ -0,0 +1,392 @@
ESEDatabaseView v1.62
Copyright (c) 2013 - 2018 Nir Sofer
Web site: http://www.nirsoft.net
Description
===========
ESEDatabaseView is a simple utility that reads and displays the data
stored inside Extensible Storage Engine (ESE) database (Also known as
'Jet Blue' or .edb file). It displays a list of all tables available in
the opened database file, allows you to choose the desired table to view,
and then when you choose a table, it displays all records found in the
selected table. ESEDatabaseView also allows you to easily choose one or
more records, and then export them into
comma-delimited/tab-delimited/html/xml file, or copy the records to the
clipboard (Ctrl+C) and then paste them into Excel or other spreadsheet
application.
System Requirements
===================
This utility works on any version of Windows, starting from Windows 2000
and up to Windows 10. Both 32-bit and 64-bit systems are supported.
esent.dll (The dll file of Extensible Storage Engine) is not required to
read the database.
Versions History
================
* Version 1.62:
o Fixed to sort date/time columns properly.
* Version 1.61:
o Added 'Run As Administrator' option (Ctrl+F11).
* Version 1.60:
o Fixed bug: On some tables ESEDatabaseView failed to read properly
some of the fields.
o Added 'Detect Ascii Strings In Binary Data' option. When it's
turned on, ESEDatabaseView displays binary data as string if it
detects that the binary data is Ascii string. This option is useful
for cookies names and values (CookieEntryEx_XX tables) in the
database of MS-Edge browser (WebCacheV01.dat).
o Added 'Put Icon On Tray' option.
* Version 1.54:
o Added new quick filter options: 'Find records with all words
(space-delimited list)' and 'Find records with all strings
(comma-delimited list)'
o Added new quick filter combo-box: 'Show only items match the
filter' and 'Hide items that match the filter'.
* Version 1.53:
o Added 'Open spartan.edb Database' which automatically opens the
spartan.edb database of IE11. This file stores the Favorites of IE11
and the full path of this file is
%LOCALAPPDATA%\Packages\Microsoft.MicrosoftEdge_8wekyb3d8bbwe\AC\Micros
oftEdge\User\Default\DataStore\Data\nouser1\120712-0049\DBStore\spartan
.edb)
o Made the display of binary data a little faster.
* Version 1.52:
o Added 'Detect UTF-16 Strings In Binary Data'. When it's turned
on, ESEDatabaseView displays binary data as string if it detects that
the binary data is UTF-16 string. (e.g: 'Key' field in MSysLocales
table)
o Fixed bug: 'Copy Selected Items' worked improperly when setting
the 'Unicode/Ascii Save Mode' to 'Always UTF-8'.
* Version 1.51:
o Fixed the 'Show Binary URL As String' feature to work properly
when the URL string starts in different position.
* Version 1.50:
o Added 'Quick Filter' feature (View -> Use Quick Filter or
Ctrl+Q). When it's turned on, you can type a string in the text-box
added under the tables combo-box and ESEDatabaseView will instantly
filter the ESE database records, showing only lines that contain the
string you typed.
* Version 1.43:
o Added 'Save All Items' (Shift+Ctrl+S).
* Version 1.42:
o Fixed bug: ESEDatabaseView crashed when using the find option
while the last item was selected.
* Version 1.41:
o Added 'Align Numeric Columns To Right' option.
* Version 1.40:
o Fixed bug: On some databases/tables (like Recipient table in
store.vol or tbUpdateLocalizedProps table in DataStore.edb)
ESEDatabaseView omitted the first 4 characters of a string.
o Added 'Select All' and 'Deselect All' buttons to the 'Choose
Column' window.
* Version 1.37:
o You can now choose the desired encoding (ANSI, UTF-8, UTF-16) to
save the csv/xml/text/html files. (Under the Options menu)
* Version 1.36:
o Added 'New ESEDatabaseView Instance' under the File menu, for
opening a new window of ESEDatabaseView.
* Version 1.35:
o When 'Auto Detect 64-bit Date/Time Value' option is turned on,
ESEDatabaseView now detects the Modified field of tbFiles table
inside DataStore.edb
o The properties window is now resizable.
* Version 1.33:
o Fixed issue: ESEDatabaseView failed to display dates earlier than
01/01/1986.
* Version 1.32:
o Added 'Show Binary URL As String'. When it's turned on, the Urls
field of tbFiles table inside DataStore.edb is displayed as string.
* Version 1.31:
o Fixed the 'Open Locked IE10/IE11 Database' option to work with
the latest build of Windows 10/IE11.
* Version 1.30:
o Added option to export to JSON file.
o Fixed bug: ESEDatabaseView failed to load records on some
tables/databases.
o Fixed bug: ESEDatabaseView crashed when trying to load a very
large binary value.
* Version 1.25:
o Fixed bug: ESEDatabaseView displayed incorrect values in
date/time fields.
* Version 1.24:
o Fixed bug: ESEDatabaseView failed to remember the last
size/position of the main window if it was not located in the primary
monitor.
* Version 1.23:
o You can now specify an empty string ("") in order to send the
data to stdout, for example:
ESEDatabaseView.exe /table "c:\temp\contacts.edb"
"SimpleContact-v081111-0122-1303" /scomma ""
* Version 1.22:
o Added 'Copy Sorted Column Data' option, which copies to the
clipboard the text of all selected items, but only the column that is
currently sorted.
* Version 1.21:
o Fixed to find the correct item when typing the string you want to
search into the main List View.
* Version 1.20:
o Added option to export all tables from command-line (Each table
in a separated file), for example:
ESEDatabaseView.exe /table "C:\temp\WebCacheV01.dat" * /scomma
"C:\Temp\export\webcache_*.csv"
* Version 1.18:
o Fixed to display local date/time values according to daylight
saving time settings.
* Version 1.17:
o Added 'Open SoftwareDistribution Database' option, which opens
the database file containing information about installed Winodws
updates (C:\WINDOWS\SoftwareDistribution\DataStore\DataStore.edb)
* Version 1.16:
o Added 'Clear Recent Files List' option.
* Version 1.15:
o Added 'Open Recent File' menu, which allows you to easily open
the last 10 database files that you previously opened.
* Version 1.10:
o Added 'Open Locked IE10 Database' option, which copies the locked
database file of Internet Explorer 10 (WebCacheV01.dat or
WebCacheV24.dat) into a temporary filename, and then opens the
temporary filename in ESEDatabaseView. You can use this option to
easily view the cache/history/cookies information stored by IE10.
* Version 1.07:
o Fixed the flickering appeared while scrolling the database
records.
* Version 1.06:
o Added 'Convert Date/Time From GMT To Local Time' option.
* Version 1.05:
o Added command-line support
* Version 1.00 - First release.
Known Limitations
=================
* Currently, ESEDatabaseView is somewhat a Beta version. It generally
reads the ESE databases properly, but in tables with complex data
structure, you may experience the following problems:
o Some fields in some of the records may display incorrect value or
display empty string while it actually contains some data.
o ESEDatabaseView may hang/stop responding when loading a table
with large amount of data.
Example for ESE Databases
=========================
ESE Databases are used by many Microsoft products. Usually, the file
extension of ESE database is .edb, but in some products the file
extension is different.
Here's some examples for .edb files used by Microsoft products:
* contacts.edb - Stores contacts information in Microsoft live products.
* WLCalendarStore.edb - Stores calendar information in Microsoft
Windows Live Mail.
* Mail.MSMessageStore - Stores messages information in Microsoft
Windows Live Mail.
* WebCacheV24.dat and WebCacheV01.dat - Stores cache, history, and
cookies information in Internet Explorer 10.
* Mailbox Database.edb and Public Folder Database.edb - Stores mail
data in Microsoft Exchange Server.
* Windows.edb - Stores index information (for Windows search) by
Windows operating system.
* DataStore.edb - Stores Windows updates information (Located under
C:\windows\SoftwareDistribution\DataStore )
* spartan.edb - Stores the Favorites of Internet Explorer 10/11.
(Stored under
%LOCALAPPDATA%\Packages\Microsoft.MicrosoftEdge_8wekyb3d8bbwe\AC\Microsof
tEdge\User\Default\DataStore\Data\nouser1\120712-0049)
Start Using ESEDatabaseView
===========================
ESEDatabaseView doesn't require any installation process or additional
dll files. In order to start using it, simple run the executable file
(ESEDatabaseView.exe) and then use the 'Open ESE Database File' option
(Ctrl+O) to open the desired .edb file. You can also drag the database
file from Explorer window into the window of ESEDatabaseView.
After opening the desired database file, the combo-box located below the
toolbar is filled with the list of all tables found in the database. By
default, MSysObjects table is selected and displayed in the main window
of ESEDatabaseView. MSysObjects is a system table available in all ESE
databases which provides the list of all tables and fields stored in the
database.
In order to view the content of another table, simply choose the desired
table in the combo-box located below the toolbar.
By default, the table is sorted according to the first column, but you
can sort by another field, simply by clicking the desired column header.
The sorting is made according to the type of the field, so... for
example, if the field is an integer value, then ESEDatabaseView will use
a numeric comparison in order to sort the column properly.
You can select one or more records (or select all records with Ctrl+A)
and then export them into text/csv/tab-delimited/html/xml file, by using
the 'Save Selected Items' option. You can also copy the selected records
into the clipboard (Ctrl+C) and then paste them (Ctrl+V) into Excel or
other spreadsheet application.
Command-Line Options
====================
/table <Database Filename> <Table Name>
Specifies the database and table to open. If the <Table Name> is "*" ,
all tables will be exported, each table in a separated file.
/stext <Filename>
Save the database table into a regular text file.
/stab <Filename>
Save the database table into a tab-delimited text file.
/scomma <Filename>
Save the database table into a comma-delimited text file (csv).
/stabular <Filename>
Save the database table into a tabular text file.
/shtml <Filename>
Save the database table into HTML file (Horizontal).
/sverhtml <Filename>
Save the database table into HTML file (Vertical).
/sxml <Filename>
Save the database table into XML file.
/sjson <Filename>
Save the database table into JSON file.
/sort <column>
This command-line option can be used with other save options for sorting
by the desired column. The <column> parameter can specify the column
index (0 for the first column, 1 for the second column, and so on) or the
name of the column, like "StatusState" and "CalculatedBuddyIdentifier".
You can specify the '~' prefix character (e.g:
"~CalculatedBuddyIdentifier") if you want to sort in descending order.
You can put multiple /sort in the command-line if you want to sort by
multiple columns.
Examples:
ESEDatabaseView.exe /table "c:\temp\contacts.edb"
"SimpleContact-v081111-0122-1303" /scomma c:\temp\1.csv
ESEDatabaseView.exe /table "c:\files\contacts.edb"
"SimpleContact-v081111-0777-1111" /shtml c:\files\1.html /Sort
"CalculatedBuddyIdentifier"
Example for exporting all tables: (Each table is exported into a
separated file)
ESEDatabaseView.exe /table "C:\temp\WebCacheV01.dat" * /scomma
"C:\Temp\export\webcache_*.csv"
The table name will replace the '*' character specified in the export
filename. For example, if the table name is Container1, then the exported
filename will be webcache_Container1.csv
Translating ESEDatabaseView to other languages
==============================================
In order to translate ESEDatabaseView to other language, follow the
instructions below:
1. Run ESEDatabaseView with /savelangfile parameter:
ESEDatabaseView.exe /savelangfile
A file named ESEDatabaseView_lng.ini will be created in the folder of
ESEDatabaseView utility.
2. Open the created language file in Notepad or in any other text
editor.
3. Translate all string entries to the desired language. Optionally,
you can also add your name and/or a link to your Web site.
(TranslatorName and TranslatorURL values) If you add this information,
it'll be used in the 'About' window.
4. After you finish the translation, Run ESEDatabaseView, and all
translated strings will be loaded from the language file.
If you want to run ESEDatabaseView without the translation, simply
rename the language file, or move it to another folder.
License
=======
This utility is released as freeware. You are allowed to freely
distribute this utility via floppy disk, CD-ROM, Internet, or in any
other way, as long as you don't charge anything for this and you don't
sell it or distribute it as a part of commercial product. If you
distribute this utility, you must include all files in the distribution
package, without any modification !
Disclaimer
==========
The software is provided "AS IS" without any warranty, either expressed
or implied, including, but not limited to, the implied warranties of
merchantability and fitness for a particular purpose. The author will not
be liable for any special, incidental, consequential or indirect damages
due to loss of data or any other reason.
Feedback
========
If you have any problem, suggestion, comment, or you found a bug in my
utility, you can send a message to nirsofer@yahoo.com