mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 18:17:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3078 from sleuthkit/release-4.5.0

Browse Source 
Merge in release-4.5.0 branch
This commit is contained in:
Richard Cordovano 2017-09-15 15:45:24 -04:00 committed by GitHub
commit 15d06c71ca
22 changed files with 1223 additions and 916 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
CentralRepository/Central Repository User Guide.pdf
View File

Binary file not shown.

15
CentralRepository/ivy.xml
View File

@ -1,15 +0,0 @@
<ivy-module version="2.0">
<info organisation="org.sleuthkit.autopsy" module="centralrepository"/>
<configurations >
<!-- module dependencies -->
<conf name="central-repository"/>
</configurations>
<dependencies>
<!-- for enterprise artifacts manager -->
<dependency conf="central-repository->default" org="org.apache.commons" name="commons-dbcp2" rev="2.1.1"/>
<dependency conf="central-repository->default" org="commons-logging" name="commons-logging" rev="1.2"/>
<dependency conf="central-repository->default" org="org.apache.commons" name="commons-pool2" rev="2.4.2"/>
<dependency conf="central-repository->default" org="org.postgresql" name="postgresql" rev="42.1.1"/>
<dependency conf="central-repository->default" org="org.xerial" name="sqlite-jdbc" rev="3.16.1"/>
</dependencies>
</ivy-module>

10
CentralRepository/ivysettings.xml
View File

@ -1,10 +0,0 @@
<ivysettings>
<settings defaultResolver="main"/>
<resolvers>
<chain name="main">
<ibiblio name="central" m2compatible="true"/>
<ibiblio name="ibiblio" m2compatible="true"/>
<ibiblio name="xerial" m2compatible="true" root="http://www.xerial.org/maven/repository/snapshot" />
</chain>
</resolvers>
</ivysettings>

7
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java Executable file → Normal file
View File

@ -33,8 +33,6 @@ import java.time.LocalDate;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import javafx.animation.KeyValue;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
@ -1062,8 +1060,9 @@ public abstract class AbstractSqlEamDb implements EamDb {
}
/**
* Sets an eamArtifact instance to the given known status. If eamArtifact
* exists, it is updated. If eamArtifact does not exist nothing happens
* Sets an eamArtifact instance to the given knownStatus. If eamArtifact
* exists, it is updated. If eamArtifact does not exist it is added
* with the given status.
*
* @param eamArtifact Artifact containing exactly one (1) ArtifactInstance.
* @param FileKnown The status to change the artifact to

70
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java
View File

@ -27,6 +27,7 @@ import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskDataException;
@ -192,4 +193,73 @@ public class EamArtifactUtil {
return null;
}
}
/**
* Create an EamArtifact from the given Content.
* Will return null if an artifact can not be created. Does not
* add the artifact to the database.
*
* @param content The content object
* @param knownStatus Unknown, known bad, or known
* @param comment The comment for the new artifact (generally used for a tag comment)
* @return The new EamArtifact or null if creation failed
*/
public static EamArtifact getEamArtifactFromContent(Content content, TskData.FileKnown knownStatus, String comment){
if(! (content instanceof AbstractFile)){
return null;
}
final AbstractFile af = (AbstractFile) content;
if ((af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
|| (af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)
|| (af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.SLACK)
|| (af.getKnown() == TskData.FileKnown.KNOWN)
|| (af.isDir() == true)
|| (!af.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC))) {
return null;
}
String dsName;
try {
dsName = af.getDataSource().getName();
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error, unable to get name of data source from abstract file.", ex);
return null;
}
// We need a hash to make the artifact
String md5 = af.getMd5Hash();
if (md5 == null || md5.isEmpty()) {
return null;
}
String deviceId;
try {
deviceId = Case.getCurrentCase().getSleuthkitCase().getDataSource(af.getDataSource().getId()).getDeviceId();
} catch (TskCoreException | TskDataException ex) {
LOGGER.log(Level.SEVERE, "Error, failed to get deviceID or data source from current case.", ex);
return null;
}
EamArtifact eamArtifact;
try {
EamArtifact.Type filesType = EamDb.getInstance().getCorrelationTypeById(EamArtifact.FILES_TYPE_ID);
eamArtifact = new EamArtifact(filesType, af.getMd5Hash());
EamArtifactInstance cei = new EamArtifactInstance(
new EamCase(Case.getCurrentCase().getName(), Case.getCurrentCase().getDisplayName()),
new EamDataSource(deviceId, dsName),
af.getParentPath() + af.getName(),
comment,
TskData.FileKnown.BAD,
EamArtifactInstance.GlobalStatus.LOCAL
);
eamArtifact.addInstance(cei);
return eamArtifact;
} catch (EamDbException ex) {
LOGGER.log(Level.SEVERE, "Error, unable to get FILES correlation type.", ex);
return null;
}
}
}

50
Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java Executable file → Normal file
View File

@ -30,11 +30,9 @@ import org.sleuthkit.autopsy.casemodule.events.BlackBoardArtifactTagDeletedEvent
import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent;
import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent;
import org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent;
import org.sleuthkit.autopsy.casemodule.services.Services;
import org.sleuthkit.autopsy.casemodule.services.TagsManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamCase;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDataSource;
@ -46,7 +44,6 @@ import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifactTag;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentTag;
import org.sleuthkit.datamodel.TagName;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskDataException;
@ -142,57 +139,14 @@ public class CaseEventListener implements PropertyChangeListener {
}
}
if ((af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
|| (af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)
|| (af.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.SLACK)
|| (af.getKnown() == TskData.FileKnown.KNOWN)
|| (af.isDir() == true)
|| (!af.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC))) {
break;
}
final EamArtifact eamArtifact = EamArtifactUtil.getEamArtifactFromContent(af,
knownStatus, comment);
String dsName;
try {
dsName = af.getDataSource().getName();
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error, unable to get name of data source from abstract file during CONTENT_TAG_ADDED event.", ex);
return;
}
String md5 = af.getMd5Hash();
if (md5 == null || md5.isEmpty()) {
return;
}
String deviceId;
try {
deviceId = Case.getCurrentCase().getSleuthkitCase().getDataSource(af.getDataSource().getId()).getDeviceId();
} catch (TskCoreException | TskDataException ex) {
LOGGER.log(Level.SEVERE, "Error, failed to get deviceID or data source from current case.", ex);
return;
}
EamArtifact eamArtifact;
try {
EamArtifact.Type filesType = dbManager.getCorrelationTypeById(EamArtifact.FILES_TYPE_ID);
eamArtifact = new EamArtifact(filesType, af.getMd5Hash());
EamArtifactInstance cei = new EamArtifactInstance(
new EamCase(Case.getCurrentCase().getName(), Case.getCurrentCase().getDisplayName()),
new EamDataSource(deviceId, dsName),
af.getParentPath() + af.getName(),
comment,
knownStatus,
EamArtifactInstance.GlobalStatus.LOCAL
);
eamArtifact.addInstance(cei);
// send update to Central Repository db
Runnable r = new KnownStatusChangeRunner(eamArtifact, knownStatus);
// TODO: send r into a thread pool instead
Thread t = new Thread(r);
t.start();
} catch (EamDbException ex) {
LOGGER.log(Level.SEVERE, "Error, unable to get FILES correlation type during CONTENT_TAG_ADDED/CONTENT_TAG_DELETED event.", ex);
}
} // CONTENT_TAG_ADDED, CONTENT_TAG_DELETED
break;

97
Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ManageTagsDialog.java
View File

@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.centralrepository.optionspanel;
import java.awt.Cursor;
import java.awt.Dimension;
import java.awt.Toolkit;
import java.util.ArrayList;
@ -27,13 +28,23 @@ import java.util.logging.Level;
import java.util.stream.Collectors;
import javax.swing.JFrame;
import javax.swing.table.DefaultTableModel;
import javax.swing.event.TableModelEvent;
import javax.swing.event.TableModelListener;
import javax.swing.JOptionPane;
import org.openide.util.NbBundle.Messages;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.services.TagsManager;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.BlackboardArtifactTag;
import org.sleuthkit.datamodel.TagName;
import org.sleuthkit.datamodel.ContentTag;
import org.sleuthkit.datamodel.TskData;
/**
* Instances of this class allow a user to select an existing hash database and
@ -92,6 +103,8 @@ final class ManageTagsDialog extends javax.swing.JDialog {
boolean enabled = badTags.contains(tagName);
model.addRow(new Object[]{tagName, enabled});
}
CheckBoxModelListener listener = new CheckBoxModelListener(this);
model.addTableModelListener(listener);
}
private void display() {
@ -231,6 +244,90 @@ final class ManageTagsDialog extends javax.swing.JDialog {
return true;
}
/**
* If the user sets a tag to "Implies known bad", give them the option to update
* any existing tagged items (in the current case only) in the central repo.
*/
public class CheckBoxModelListener implements TableModelListener {
@Messages({"ManageTagsDialog.updateCurrentCase.msg=Mark as known bad any files/artifacts in the current case that have this tag?",
"ManageTagsDialog.updateCurrentCase.title=Update current case?",
"ManageTagsDialog.updateCurrentCase.error=Error updating existing Central Repository entries"})
javax.swing.JDialog dialog;
public CheckBoxModelListener(javax.swing.JDialog dialog){
this.dialog = dialog;
}
@Override
public void tableChanged(TableModelEvent e) {
int row = e.getFirstRow();
int column = e.getColumn();
if (column == 1) {
DefaultTableModel model = (DefaultTableModel) e.getSource();
String tagName = (String) model.getValueAt(row, 0);
Boolean checked = (Boolean) model.getValueAt(row, column);
if (checked) {
// Don't do anything if there's no case open
if(Case.isCaseOpen()){
int dialogButton = JOptionPane.YES_NO_OPTION;
int dialogResult = JOptionPane.showConfirmDialog (
null,
Bundle.ManageTagsDialog_updateCurrentCase_msg(),
Bundle.ManageTagsDialog_updateCurrentCase_title(),
dialogButton);
if(dialogResult == JOptionPane.YES_OPTION){
try{
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
setArtifactsKnownBadByTag(tagName, Case.getCurrentCase());
} catch (EamDbException ex) {
LOGGER.log(Level.SEVERE, "Failed to apply known bad status to current case", ex);
JOptionPane.showMessageDialog(null, Bundle.ManageTagsDialog_updateCurrentCase_error());
} finally {
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
}
}
}
}
}
}
}
/**
* Set knownBad status for all files/artifacts in the given case that
* are tagged with the given tag name.
* Files/artifacts that are not already in the database will be added.
* @param tagName The name of the tag to search for
* @param curCase The case to search in
*/
public void setArtifactsKnownBadByTag(String tagNameString, Case curCase) throws EamDbException{
try{
TagName tagName = curCase.getServices().getTagsManager().getDisplayNamesToTagNamesMap().get(tagNameString);
// First find any matching artifacts
List<BlackboardArtifactTag> artifactTags = curCase.getSleuthkitCase().getBlackboardArtifactTagsByTagName(tagName);
for(BlackboardArtifactTag bbTag:artifactTags){
List<EamArtifact> convertedArtifacts = EamArtifactUtil.fromBlackboardArtifact(bbTag.getArtifact(), true,
EamDb.getInstance().getCorrelationTypes(), true);
for (EamArtifact eamArtifact : convertedArtifacts) {
EamDb.getInstance().setArtifactInstanceKnownStatus(eamArtifact,TskData.FileKnown.BAD);
}
}
// Now search for files
List<ContentTag> fileTags = curCase.getSleuthkitCase().getContentTagsByTagName(tagName);
for(ContentTag contentTag:fileTags){
final EamArtifact eamArtifact = EamArtifactUtil.getEamArtifactFromContent(contentTag.getContent(),
TskData.FileKnown.BAD, "");
EamDb.getInstance().setArtifactInstanceKnownStatus(eamArtifact, TskData.FileKnown.BAD);
}
} catch (TskCoreException ex){
throw new EamDbException("Error updating artifacts", ex);
}
}
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.ButtonGroup buttonGroup1;
private javax.swing.JButton cancelButton;

44
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java
View File

@ -304,6 +304,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
* text box.
*/
@Messages({
"# {0} - case db status", "# {1} - search svc Status", "# {2} - coord svc Status", "# {3} - msg broker status",
"AutoIngestControlPanel.tbServicesStatusMessage.Message=Case databases {0}, keyword search {1}, coordination {2}, messaging {3} ",
"AutoIngestControlPanel.tbServicesStatusMessage.Message.Up=up",
"AutoIngestControlPanel.tbServicesStatusMessage.Message.Down=down",
@ -669,8 +670,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
@Messages({
"AutoIngestControlPanel.AutoIngestStartupError=Failed to start automated ingest. Verify Multi-user Settings.",
"AutoIngestControlPanel.AutoIngestStartupFailed.Message=Failed to start automated ingest.\nPlease see auto ingest system log for details.",
"AutoIngestControlPanel.AutoIngestStartupFailed.Title=Automated Ingest Error",
})
"AutoIngestControlPanel.AutoIngestStartupFailed.Title=Automated Ingest Error",})
private void startUp() {
/*
@ -679,7 +679,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
try {
manager.startUp();
autoIngestStarted = true;
} catch (AutoIngestManager.AutoIngestManagerStartupException ex) {
} catch (AutoIngestManager.AutoIngestManagerException ex) {
SYS_LOGGER.log(Level.SEVERE, "Dashboard error starting up auto ingest", ex);
tbStatusMessage.setText(NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.AutoIngestStartupError"));
manager = null;
@ -812,8 +812,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
"AutoIngestControlPanel.PauseDueToWriteStateFilesFailure=Paused, unable to write to shared images or cases location.",
"AutoIngestControlPanel.PauseDueToSharedConfigError=Paused, unable to update shared configuration.",
"AutoIngestControlPanel.PauseDueToIngestJobStartFailure=Paused, unable to start ingest job processing.",
"AutoIngestControlPanel.PauseDueToFileExporterError=Paused, unable to load File Exporter settings.",
})
"AutoIngestControlPanel.PauseDueToFileExporterError=Paused, unable to load File Exporter settings.",})
@Override
public void update(Observable o, Object arg) {
@ -983,7 +982,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
List<AutoIngestJob> completedJobs = new ArrayList<>();
manager.getJobs(pendingJobs, runningJobs, completedJobs);
// Sort the completed jobs list by completed date
Collections.sort(completedJobs, new AutoIngestJob.ReverseDateCompletedComparator());
Collections.sort(completedJobs, new AutoIngestJob.ReverseCompletedDateComparator());
EventQueue.invokeLater(new RefreshComponentsTask(pendingJobs, runningJobs, completedJobs));
}
}
@ -1076,7 +1075,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
* @return True or fale.
*/
private boolean isLocalJob(AutoIngestJob job) {
return job.getNodeName().equals(LOCAL_HOST_NAME);
return job.getProcessingHostName().equals(LOCAL_HOST_NAME);
}
/**
@ -1145,20 +1144,19 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
tableModel.setRowCount(0);
for (AutoIngestJob job : jobs) {
AutoIngestJob.StageDetails status = job.getStageDetails();
AutoIngestJobNodeData nodeData = job.getNodeData();
tableModel.addRow(new Object[]{
nodeData.getCaseName(), // CASE
nodeData.getDataSourcePath().getFileName(), // DATA_SOURCE
job.getNodeName(), // HOST_NAME
nodeData.getManifestFileDate(), // CREATED_TIME
job.getStageStartDate(), // STARTED_TIME
nodeData.getCompletedDate(), // COMPLETED_TIME
job.getManifest().getCaseName(), // CASE
job.getManifest().getDataSourcePath().getFileName(), // DATA_SOURCE
job.getProcessingHostName(), // HOST_NAME
job.getManifest().getDateFileCreated(), // CREATED_TIME
job.getProcessingStageStartDate(), // STARTED_TIME
job.getCompletedDate(), // COMPLETED_TIME
status.getDescription(), // ACTIVITY
nodeData.getErrorsOccurred(), // STATUS
job.getErrorsOccurred(), // STATUS
((Date.from(Instant.now()).getTime()) - (status.getStartDate().getTime())), // ACTIVITY_TIME
job.getCaseDirectoryPath(), // CASE_DIRECTORY_PATH
job.getNodeName().equals(LOCAL_HOST_NAME), // IS_LOCAL_JOB
nodeData.getManifestFilePath()}); // MANIFEST_FILE_PATH
job.getProcessingHostName().equals(LOCAL_HOST_NAME), // IS_LOCAL_JOB
job.getManifest().getFilePath()}); // MANIFEST_FILE_PATH
}
} catch (Exception ex) {
SYS_LOGGER.log(Level.SEVERE, "Dashboard error refreshing table", ex);
@ -1703,11 +1701,17 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
*
* @param evt The button click event.
*/
@Messages({"AutoIngestControlPanel.casePrioritization.errorMessage=An error occurred when prioritizing the case. Some or all jobs may not have been prioritized."})
private void bnPrioritizeCaseActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_bnPrioritizeCaseActionPerformed
if (pendingTableModel.getRowCount() > 0 && pendingTable.getSelectedRow() >= 0) {
this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
String caseName = (pendingTableModel.getValueAt(pendingTable.getSelectedRow(), JobsTableModelColumns.CASE.ordinal())).toString();
try {
manager.prioritizeCase(caseName);
} catch (AutoIngestManager.AutoIngestManagerException ex) {
SYS_LOGGER.log(Level.SEVERE, "Error prioritizing a case", ex);
MessageNotifyUtil.Message.error(Bundle.AutoIngestControlPanel_casePrioritization_errorMessage());
}
refreshTables();
pendingTable.clearSelection();
enablePendingTableButtons(false);
@ -1756,11 +1760,17 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
}
}//GEN-LAST:event_bnShowCaseLogActionPerformed
@Messages({"AutoIngestControlPanel.jobPrioritization.errorMessage=An error occurred when prioritizing the job."})
private void bnPrioritizeJobActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_bnPrioritizeJobActionPerformed
if (pendingTableModel.getRowCount() > 0 && pendingTable.getSelectedRow() >= 0) {
this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
Path manifestFilePath = (Path) (pendingTableModel.getValueAt(pendingTable.getSelectedRow(), JobsTableModelColumns.MANIFEST_FILE_PATH.ordinal()));
try {
manager.prioritizeJob(manifestFilePath);
} catch (AutoIngestManager.AutoIngestManagerException ex) {
SYS_LOGGER.log(Level.SEVERE, "Error prioritizing a case", ex);
MessageNotifyUtil.Message.error(Bundle.AutoIngestControlPanel_jobPrioritization_errorMessage());
}
refreshTables();
pendingTable.clearSelection();
enablePendingTableButtons(false);

11
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.form
View File

@ -1,6 +1,15 @@
<?xml version="1.0" encoding="UTF-8" ?>
<Form version="1.6" maxVersion="1.9" type="org.netbeans.modules.form.forminfo.JPanelFormInfo">
<NonVisualComponents>
<Component class="javax.swing.JButton" name="jButton1">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties" key="AutoIngestDashboard.jButton1.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
</NonVisualComponents>
<AuxValues>
<AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
@ -41,7 +50,7 @@
<Component id="runningScrollPane" alignment="0" max="32767" attributes="0"/>
<Component id="completedScrollPane" alignment="0" max="32767" attributes="0"/>
</Group>
<EmptySpace max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>

39
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java
View File

@ -31,7 +31,6 @@ import java.util.logging.Level;
import javax.swing.DefaultListSelectionModel;
import java.awt.Color;
import java.beans.PropertyChangeEvent;
import java.util.Collections;
import javax.swing.JPanel;
import javax.swing.JTable;
import javax.swing.SwingWorker;
@ -438,11 +437,9 @@ public final class AutoIngestDashboard extends JPanel implements Observer {
List<AutoIngestJob> pendingJobs = jobsSnapshot.getPendingJobs();
List<AutoIngestJob> runningJobs = jobsSnapshot.getRunningJobs();
List<AutoIngestJob> completedJobs = jobsSnapshot.getCompletedJobs();
// DLG: DONE! Do the appropriate sorts for each table.
Collections.sort(pendingJobs, new AutoIngestJob.PriorityComparator());
runningJobs.sort(new AutoIngestJob.AlphabeticalComparator());
pendingJobs.sort(new AutoIngestJob.PriorityComparator());
runningJobs.sort(new AutoIngestJob.CaseNameAndProcessingHostComparator());
completedJobs.sort(new AutoIngestJob.ReverseCompletedDateComparator());
refreshTable(pendingJobs, pendingTable, pendingTableModel);
refreshTable(runningJobs, runningTable, runningTableModel);
refreshTable(completedJobs, completedTable, completedTableModel);
@ -462,27 +459,25 @@ public final class AutoIngestDashboard extends JPanel implements Observer {
Path currentRow = getSelectedEntry(table, tableModel);
tableModel.setRowCount(0);
for (AutoIngestJob job : jobs) {
if (job.getNodeData().getVersion() < 1) {
if (job.getVersion() < 1) {
// Ignore version '0' nodes since they don't carry enough
// data to populate the table.
continue;
}
AutoIngestJob.StageDetails status = job.getStageDetails();
AutoIngestJobNodeData nodeData = job.getNodeData();
tableModel.addRow(new Object[]{
nodeData.getCaseName(), // CASE
nodeData.getDataSourcePath().getFileName(), // DATA_SOURCE
job.getNodeName(), // HOST_NAME
nodeData.getManifestFileDate(), // CREATED_TIME
job.getStageStartDate(), // STARTED_TIME
nodeData.getCompletedDate(), // COMPLETED_TIME
job.getManifest().getCaseName(), // CASE
job.getManifest().getDataSourcePath().getFileName(), job.getProcessingHostName(), // HOST_NAME
job.getManifest().getDateFileCreated(), // CREATED_TIME
job.getProcessingStageStartDate(), // STARTED_TIME
job.getCompletedDate(), // COMPLETED_TIME
status.getDescription(), // ACTIVITY
nodeData.getErrorsOccurred(), // STATUS
job.getErrorsOccurred(), // STATUS
((Date.from(Instant.now()).getTime()) - (status.getStartDate().getTime())), // ACTIVITY_TIME
job.getCaseDirectoryPath(), // CASE_DIRECTORY_PATH
nodeData.getManifestFilePath()//DLG: , // MANIFEST_FILE_PATH
//DLG: job
}); // JOB
job.getManifest().getFilePath() // MANIFEST_FILE_PATH
//DLG: Put job object in the table
});
}
setSelectedEntry(table, tableModel, currentRow);
} catch (Exception ex) {
@ -547,7 +542,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer {
*/
private enum JobsTableModelColumns {
// DLG: Go through the bundles.properties file and delete any unused key-value pairs.
// DLG: Go through the bundle.properties file and delete any unused key-value pairs.
CASE(NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.JobsTableModel.ColumnHeader.Case")),
DATA_SOURCE(NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.JobsTableModel.ColumnHeader.ImageFolder")),
HOST_NAME(NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.JobsTableModel.ColumnHeader.HostName")),
@ -662,6 +657,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer {
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
jButton1 = new javax.swing.JButton();
pendingScrollPane = new javax.swing.JScrollPane();
pendingTable = new javax.swing.JTable();
runningScrollPane = new javax.swing.JScrollPane();
@ -676,6 +672,8 @@ public final class AutoIngestDashboard extends JPanel implements Observer {
tbServicesStatusMessage = new javax.swing.JTextField();
prioritizeButton = new javax.swing.JButton();
org.openide.awt.Mnemonics.setLocalizedText(jButton1, org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.jButton1.text")); // NOI18N
pendingTable.setModel(pendingTableModel);
pendingTable.setToolTipText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.pendingTable.toolTipText")); // NOI18N
pendingTable.setAutoResizeMode(javax.swing.JTable.AUTO_RESIZE_ALL_COLUMNS);
@ -843,8 +841,6 @@ public final class AutoIngestDashboard extends JPanel implements Observer {
jobsSnapshot = autoIngestMonitor.prioritizeJob(manifestFilePath);
refreshTables(jobsSnapshot);
} catch (AutoIngestMonitor.AutoIngestMonitorException ex) {
// DLG: DONE! Log the exception and do a popup with a user-friendly
// message explaining that the operation failed
String errorMessage = String.format(NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.PrioritizeError"), manifestFilePath);
logger.log(Level.SEVERE, errorMessage, ex);
MessageNotifyUtil.Message.error(errorMessage);
@ -856,6 +852,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer {
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JScrollPane completedScrollPane;
private javax.swing.JTable completedTable;
private javax.swing.JButton jButton1;
private javax.swing.JLabel lbCompleted;
private javax.swing.JLabel lbPending;
private javax.swing.JLabel lbRunning;

280
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2015-2017 Basis Technology Corp.
* Copyright 2011-2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -22,6 +22,7 @@ import java.io.Serializable;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.Objects;
@ -40,23 +41,63 @@ import org.sleuthkit.autopsy.ingest.IngestJob;
public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializable {
private static final long serialVersionUID = 1L;
private static final int CURRENT_VERSION = 1;
private static final int DEFAULT_PRIORITY = 0;
private static final String LOCAL_HOST_NAME = NetworkUtils.getLocalHostName();
private final AutoIngestJobNodeData nodeData;
private final int version;
private final Manifest manifest;
private final String nodeName;
@GuardedBy("this")
private String caseDirectoryPath; // DLG: Replace with AutoIngestJobNodeData.caseDirectoryPath
private String caseDirectoryPath;
@GuardedBy("this")
private Integer priority;
@GuardedBy("this")
private Stage stage;
@GuardedBy("this")
private Date stageStartDate;
@GuardedBy("this")
private StageDetails stageDetails;
@GuardedBy("this")
transient private DataSourceProcessor dataSourceProcessor;
@GuardedBy("this")
transient private IngestJob ingestJob;
@GuardedBy("this")
transient private boolean canceled;
transient private boolean cancelled;
@GuardedBy("this")
transient private boolean completed;
@GuardedBy("this")
private Date completedDate;
@GuardedBy("this")
private boolean errorsOccurred;
@GuardedBy("this")
private ProcessingStatus processingStatus;
@GuardedBy("this")
private int numberOfCrashes;
/**
* Constructs a new automated ingest job for a manifest. All job state not
* specified in the manifest is set to the default state for a new job.
*
* @param manifest The manifest.
*/
AutoIngestJob(Manifest manifest) {
this.version = CURRENT_VERSION;
this.manifest = manifest;
this.nodeName = AutoIngestJob.LOCAL_HOST_NAME;
this.caseDirectoryPath = "";
this.priority = DEFAULT_PRIORITY;
this.stage = Stage.PENDING;
this.stageStartDate = manifest.getDateFileCreated();
this.stageDetails = this.getStageDetails();
this.dataSourceProcessor = null;
this.ingestJob = null;
this.cancelled = false;
this.completed = false;
this.completedDate = new Date(0);
this.errorsOccurred = false;
this.processingStatus = ProcessingStatus.PENDING;
this.numberOfCrashes = 0;
}
/**
* Constructs an automated ingest job for a manifest. The manifest specifies
@ -67,48 +108,33 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
* AutoIngestJob class.
*
* @param nodeData The node data.
* @param caseDirectoryPath The path to the case directory for the job, may
* be null.
* @param nodeName If the job is in progress, the node doing the
* processing, otherwise the locla host.
* @param stage The processing stage for display purposes.
*/
/*
* DLG: We need a contrucotr that takes just the node data. When we have
* added the case dierectory path, the host name and the stage data to the
* ZK nodes, we probably cna use that constructor only. I'm thinking this
* because we will creater node data with initial values when we first
* discover the nodes, and then we will continue to update it.
*/
AutoIngestJob(AutoIngestJobNodeData nodeData, Path caseDirectoryPath, String nodeName, Stage stage) {
this.nodeData = nodeData;
if (null != caseDirectoryPath) {
this.caseDirectoryPath = caseDirectoryPath.toString();
} else {
this.caseDirectoryPath = "";
}
this.nodeName = nodeName;
this.stage = stage;
this.stageStartDate = nodeData.getManifestFileDate();
AutoIngestJob(AutoIngestJobNodeData nodeData) {
this.version = nodeData.getVersion();
this.manifest = new Manifest(nodeData.getManifestFilePath(), nodeData.getManifestFileDate(), nodeData.getCaseName(), nodeData.getDeviceId(), nodeData.getDataSourcePath(), Collections.emptyMap());
this.nodeName = nodeData.getProcessingHostName();
this.caseDirectoryPath = nodeData.getCaseDirectoryPath().toString();
this.priority = nodeData.getPriority();
this.stage = nodeData.getProcessingStage();
this.stageStartDate = nodeData.getProcessingStageStartDate();
this.stageDetails = this.getStageDetails();
this.dataSourceProcessor = null;
this.ingestJob = null;
this.cancelled = false;
this.completed = false;
this.completedDate = nodeData.getCompletedDate();
this.errorsOccurred = nodeData.getErrorsOccurred();
this.processingStatus = nodeData.getProcessingStatus();
this.numberOfCrashes = nodeData.getNumberOfCrashes();
}
/**
* Gets the auto ingest job node data.
* Gets the auto ingest job manifest.
*
* @return The node data.
* @return The manifest.
*/
AutoIngestJobNodeData getNodeData() {
return this.nodeData;
}
/**
* Queries whether or not a case directory path has been set for this auto
* ingest job.
*
* @return True or false
*/
synchronized boolean hasCaseDirectoryPath() {
return (false == this.caseDirectoryPath.isEmpty());
Manifest getManifest() {
return this.manifest;
}
/**
@ -135,30 +161,46 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
}
}
synchronized void setStage(Stage newStage) {
setStage(newStage, Date.from(Instant.now()));
/**
* Sets the priority of the job. A higher number indicates a higher
* priority.
*
* @param priority The priority.
*/
synchronized void setPriority(Integer priority) {
this.priority = priority;
}
synchronized void setStage(Stage newState, Date stateStartedDate) {
if (Stage.CANCELING == this.stage && Stage.COMPLETED != newState) {
/**
* Gets the priority of the job. A higher number indicates a higher
* priority.
*
* @return The priority.
*/
synchronized Integer getPriority() {
return this.priority;
}
synchronized void setStage(Stage newStage) {
if (Stage.CANCELLING == this.stage && Stage.COMPLETED != newStage) {
return;
}
this.stage = newState;
this.stageStartDate = stateStartedDate;
this.stage = newStage;
this.stageStartDate = Date.from(Instant.now());
}
synchronized Stage getStage() {
synchronized Stage getProcessingStage() {
return this.stage;
}
synchronized Date getStageStartDate() {
return this.stageStartDate;
synchronized Date getProcessingStageStartDate() {
return new Date(this.stageStartDate.getTime());
}
synchronized StageDetails getStageDetails() {
String description;
Date startDate;
if (Stage.CANCELING != this.stage && null != this.ingestJob) {
if (Stage.CANCELLING != this.stage && null != this.ingestJob) {
IngestJob.ProgressSnapshot progress = this.ingestJob.getSnapshot();
IngestJob.DataSourceIngestModuleHandle ingestModuleHandle = progress.runningDataSourceIngestModule();
if (null != ingestModuleHandle) {
@ -171,7 +213,7 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
if (!ingestModuleHandle.isCancelled()) {
description = ingestModuleHandle.displayName();
} else {
description = String.format(Stage.CANCELING_MODULE.getDisplayText(), ingestModuleHandle.displayName());
description = String.format(Stage.CANCELLING_MODULE.getDisplayText(), ingestModuleHandle.displayName());
}
} else {
/**
@ -189,7 +231,12 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
description = this.stage.getDisplayText();
startDate = this.stageStartDate;
}
return new StageDetails(description, startDate);
this.stageDetails = new StageDetails(description, startDate);
return this.stageDetails;
}
synchronized void setStageDetails(StageDetails stageDetails) {
this.stageDetails = stageDetails;
}
synchronized void setDataSourceProcessor(DataSourceProcessor dataSourceProcessor) {
@ -205,9 +252,9 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
}
synchronized void cancel() {
setStage(Stage.CANCELING);
canceled = true;
nodeData.setErrorsOccurred(true);
setStage(Stage.CANCELLING);
cancelled = true;
errorsOccurred = true;
if (null != dataSourceProcessor) {
dataSourceProcessor.cancel();
}
@ -217,7 +264,7 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
}
synchronized boolean isCanceled() {
return canceled;
return cancelled;
}
synchronized void setCompleted() {
@ -229,10 +276,68 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
return completed;
}
String getNodeName() {
/**
* Sets the date the job was completed, with or without cancellation or
* errors.
*
* @param completedDate The completion date.
*/
synchronized void setCompletedDate(Date completedDate) {
this.completedDate = new Date(completedDate.getTime());
}
/**
* Gets the date the job was completed, with or without cancellation or
* errors.
*
* @return True or false.
*/
synchronized Date getCompletedDate() {
return new Date(completedDate.getTime());
}
/**
* Sets whether or not errors occurred during the processing of the job.
*
* @param errorsOccurred True or false;
*/
synchronized void setErrorsOccurred(boolean errorsOccurred) {
this.errorsOccurred = errorsOccurred;
}
/**
* Queries whether or not errors occurred during the processing of the job.
*
* @return True or false.
*/
synchronized boolean getErrorsOccurred() {
return this.errorsOccurred;
}
synchronized String getProcessingHostName() {
return nodeName;
}
int getVersion() {
return this.version;
}
synchronized ProcessingStatus getProcessingStatus() {
return this.processingStatus;
}
synchronized void setProcessingStatus(ProcessingStatus processingStatus) {
this.processingStatus = processingStatus;
}
synchronized int getNumberOfCrashes() {
return this.numberOfCrashes;
}
synchronized void setNumberOfCrashes(int numberOfCrashes) {
this.numberOfCrashes = numberOfCrashes;
}
@Override
public boolean equals(Object obj) {
if (!(obj instanceof AutoIngestJob)) {
@ -241,11 +346,7 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
if (obj == this) {
return true;
}
Path manifestPath1 = this.getNodeData().getManifestFilePath();
Path manifestPath2 = ((AutoIngestJob) obj).getNodeData().getManifestFilePath();
return manifestPath1.equals(manifestPath2);
return this.getManifest().getFilePath().equals(((AutoIngestJob) obj).getManifest().getFilePath());
}
@Override
@ -256,29 +357,20 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
@Override
public int compareTo(AutoIngestJob o) {
Date date1 = this.getNodeData().getManifestFileDate();
Date date2 = o.getNodeData().getManifestFileDate();
return -date1.compareTo(date2);
}
// DLG: Add a toString override
@Override
public String toString() {
// DLG: FINISH ME!
return "";
return -this.getManifest().getDateFileCreated().compareTo(o.getManifest().getDateFileCreated());
}
/**
* Custom comparator that allows us to sort List<AutoIngestJob> on reverse
* chronological date modified (descending)
*/
static class ReverseDateCompletedComparator implements Comparator<AutoIngestJob> {
static class ReverseCompletedDateComparator implements Comparator<AutoIngestJob> {
@Override
public int compare(AutoIngestJob o1, AutoIngestJob o2) {
return -o1.getStageStartDate().compareTo(o2.getStageStartDate());
return -o1.getCompletedDate().compareTo(o2.getCompletedDate());
}
}
/**
@ -288,10 +380,7 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
@Override
public int compare(AutoIngestJob job, AutoIngestJob anotherJob) {
Integer priority1 = job.getNodeData().getPriority();
Integer priority2 = anotherJob.getNodeData().getPriority();
return -priority1.compareTo(priority2);
return -(job.getPriority().compareTo(anotherJob.getPriority()));
}
}
@ -301,21 +390,29 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
* alphabetically except for jobs for the current host, which are placed at
* the top of the list.
*/
static class AlphabeticalComparator implements Comparator<AutoIngestJob> {
static class CaseNameAndProcessingHostComparator implements Comparator<AutoIngestJob> {
@Override
public int compare(AutoIngestJob o1, AutoIngestJob o2) {
if (o1.getNodeName().equalsIgnoreCase(LOCAL_HOST_NAME)) {
if (o1.getProcessingHostName().equalsIgnoreCase(LOCAL_HOST_NAME)) {
return -1; // o1 is for current case, float to top
} else if (o2.getNodeName().equalsIgnoreCase(LOCAL_HOST_NAME)) {
} else if (o2.getProcessingHostName().equalsIgnoreCase(LOCAL_HOST_NAME)) {
return 1; // o2 is for current case, float to top
} else {
String caseName1 = o1.getNodeData().getCaseName();
String caseName2 = o2.getNodeData().getCaseName();
return o1.getManifest().getCaseName().compareToIgnoreCase(o2.getManifest().getCaseName());
}
}
return caseName1.compareToIgnoreCase(caseName2);
}
}
/**
* Processing status for the auto ingest job for the manifest.
*/
enum ProcessingStatus {
PENDING,
PROCESSING,
COMPLETED,
DELETED
}
enum Stage {
@ -330,8 +427,8 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
ANALYZING_DATA_SOURCE("Analyzing data source"),
ANALYZING_FILES("Analyzing files"),
EXPORTING_FILES("Exporting files"),
CANCELING_MODULE("Canceling module"),
CANCELING("Canceling"),
CANCELLING_MODULE("Cancelling module"),
CANCELLING("Cancelling"),
COMPLETED("Completed");
private final String displayText;
@ -347,12 +444,13 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
}
@Immutable
static final class StageDetails {
static final class StageDetails implements Serializable {
private static final long serialVersionUID = 1L;
private final String description;
private final Date startDate;
private StageDetails(String description, Date startDate) {
StageDetails(String description, Date startDate) {
this.description = description;
this.startDate = startDate;
}
@ -362,7 +460,7 @@ public final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializa
}
Date getStartDate() {
return this.startDate;
return new Date(this.startDate.getTime());
}
}

447
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobNodeData.java
View File

@ -18,7 +18,7 @@
*/
package org.sleuthkit.autopsy.experimental.autoingest;
import java.io.Serializable;
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -26,20 +26,19 @@ import java.util.Date;
import javax.lang.model.type.TypeKind;
/**
* A coordination service node data transfer object for an auto ingest job.
* An object that converts auto ingest job data for an auto ingest job
* coordination service node to and from byte arrays.
*/
final class AutoIngestJobNodeData implements Serializable {
final class AutoIngestJobNodeData {
private static final long serialVersionUID = 1L;
private static final int NODE_DATA_VERSION = 1;
private static final int MAX_POSSIBLE_NODE_DATA_SIZE = 131493;
private static final int CURRENT_VERSION = 1;
private static final int MAX_POSSIBLE_NODE_DATA_SIZE = 131629;
private static final int DEFAULT_PRIORITY = 0;
/*
* Version 0 fields.
*/
private final boolean coordSvcNodeDataWasSet;
private ProcessingStatus status;
private int processingStatus;
private int priority;
private int numberOfCrashes;
private long completedDate;
@ -49,228 +48,197 @@ final class AutoIngestJobNodeData implements Serializable {
* Version 1 fields.
*/
private int version;
private String deviceId;
private String caseName;
private String caseDirectoryPath;
private long manifestFileDate;
private String manifestFilePath;
private long manifestFileDate;
private String caseName;
private String deviceId;
private String dataSourcePath;
private String processingStage;
private String caseDirectoryPath;
private String processingHostName;
private byte processingStage;
private long processingStageStartDate;
private String processingHost;
private String processingStageDetailsDescription;
private long processingStageDetailsStartDate;
//DLG: Add caseDirectoryPath from AutoIngestJob
/*
* DLG: Rename class to AutoIngestJobNodeData - Add String
* caseDirectoryPath. Needed to locate case auto ingest log and later, for
* case deletion
*
* Add String processingStage, long processingStageStartDate, String
* processingHost fields. These three fields are needed to populate running
* jobs table; use of auto ingest job data is not enough, because there
* would be no data until a status event was received by the auto ingest
* monitor.
*
* Update the AutoIngestManager code that creates ZK nodes for auto ingest
* jobs to write the new fields described above to new nodes
*
* Update the AutoIngestManager code that publishes auto ingest status
* events for the current job to update the the processing status fields
* described above in addition to publishing AutoIngestJobStatusEvents.
* Probably also need to write this data initially when a jo becomes the
* current job.
*/
/**
* Constructs a coordination service node data data transfer object for an
* auto ingest manifest from the raw bytes obtained from the coordination
* service.
* Uses an auto ingest job to construct an object that converts auto ingest
* job data for an auto ingest job coordination service node to and from
* byte arrays.
*
* @param job The job.
*/
AutoIngestJobNodeData(AutoIngestJob job) {
setProcessingStatus(job.getProcessingStatus());
setPriority(job.getPriority());
setNumberOfCrashes(numberOfCrashes); // RJCTODO
setCompletedDate(job.getCompletedDate());
setErrorsOccurred(job.getErrorsOccurred());
this.version = CURRENT_VERSION;
Manifest manifest = job.getManifest();
setManifestFilePath(manifest.getFilePath());
setManifestFileDate(manifest.getDateFileCreated());
setCaseName(manifest.getCaseName());
setDeviceId(manifest.getDeviceId());
setDataSourcePath(manifest.getDataSourcePath());
setCaseDirectoryPath(job.getCaseDirectoryPath());
setProcessingHostName(job.getProcessingHostName());
setProcessingStage(job.getProcessingStage());
setProcessingStageStartDate(job.getProcessingStageStartDate());
setProcessingStageDetails(job.getStageDetails());
}
/**
* Uses a coordination service node data to construct an object that
* converts auto ingest job data for an auto ingest job coordination service
* node to and from byte arrays.
*
* @param nodeData The raw bytes received from the coordination service.
*/
AutoIngestJobNodeData(byte[] nodeData) throws AutoIngestJobNodeDataException {
ByteBuffer buffer = ByteBuffer.wrap(nodeData);
this.coordSvcNodeDataWasSet = buffer.hasRemaining();
if (this.coordSvcNodeDataWasSet) {
int rawStatus = buffer.getInt();
if (ProcessingStatus.PENDING.ordinal() == rawStatus) {
this.status = ProcessingStatus.PENDING;
} else if (ProcessingStatus.PROCESSING.ordinal() == rawStatus) {
this.status = ProcessingStatus.PROCESSING;
} else if (ProcessingStatus.COMPLETED.ordinal() == rawStatus) {
this.status = ProcessingStatus.COMPLETED;
} else if (ProcessingStatus.DELETED.ordinal() == rawStatus) {
this.status = ProcessingStatus.DELETED;
AutoIngestJobNodeData(byte[] nodeData) throws InvalidDataException {
if (null == nodeData || nodeData.length == 0) {
throw new InvalidDataException(null == nodeData ? "Null nodeData byte array" : "Zero-length nodeData byte array");
}
/*
* Set default values for all fields.
*/
this.processingStatus = AutoIngestJob.ProcessingStatus.PENDING.ordinal();
this.priority = DEFAULT_PRIORITY;
this.numberOfCrashes = 0;
this.completedDate = 0L;
this.errorsOccurred = false;
this.version = CURRENT_VERSION;
this.manifestFilePath = "";
this.manifestFileDate = 0L;
this.caseName = "";
this.deviceId = "";
this.dataSourcePath = "";
this.caseDirectoryPath = "";
this.processingHostName = "";
this.processingStage = (byte) AutoIngestJob.Stage.PENDING.ordinal();
this.processingStageStartDate = 0L;
this.processingStageDetailsDescription = "";
this.processingStageDetailsStartDate = 0L;
/*
* Get fields from node data.
*/
ByteBuffer buffer = ByteBuffer.wrap(nodeData);
try {
if (buffer.hasRemaining()) {
/*
* Get version 0 fields.
*/
this.processingStatus = buffer.getInt();
this.priority = buffer.getInt();
this.numberOfCrashes = buffer.getInt();
this.completedDate = buffer.getLong();
int errorFlag = buffer.getInt();
this.errorsOccurred = (1 == errorFlag);
} else {
this.status = ProcessingStatus.PENDING;
this.priority = DEFAULT_PRIORITY;
this.numberOfCrashes = 0;
this.completedDate = 0L;
this.errorsOccurred = false;
}
if (buffer.hasRemaining()) {
/*
* There are more than 24 bytes in the buffer, so we assume the
* version is greater than '0'.
* Get version 1 fields.
*/
this.version = buffer.getInt();
if (this.version > NODE_DATA_VERSION) {
throw new AutoIngestJobNodeDataException(String.format("Node data version %d is not suppored.", this.version));
}
this.deviceId = getStringFromBuffer(buffer, TypeKind.BYTE);
this.caseName = getStringFromBuffer(buffer, TypeKind.BYTE);
//DLG: this.caseDirectoryPath = getStringFromBuffer(buffer, TypeKind.SHORT);
this.caseDirectoryPath = getStringFromBuffer(buffer, TypeKind.SHORT);
this.manifestFileDate = buffer.getLong();
this.manifestFilePath = getStringFromBuffer(buffer, TypeKind.SHORT);
this.dataSourcePath = getStringFromBuffer(buffer, TypeKind.SHORT);
//DLG: this.processingStage = getStringFromBuffer(buffer, TypeKind.BYTE);
//DLG: this.processingStageStartDate = buffer.getLong();
//DLG: this.processingHost = getStringFromBuffer(buffer, TypeKind.SHORT);
} else {
this.version = 0;
this.deviceId = "";
this.caseName = "";
this.caseDirectoryPath = "";
this.manifestFileDate = 0L;
this.manifestFilePath = "";
this.dataSourcePath = "";
this.processingStage = "";
this.processingStageStartDate = 0L;
this.processingHost = "";
this.processingStage = buffer.get();
this.processingStageStartDate = buffer.getLong();
this.processingStageDetailsDescription = getStringFromBuffer(buffer, TypeKind.BYTE);
this.processingStageDetailsStartDate = buffer.getLong();;
this.processingHostName = getStringFromBuffer(buffer, TypeKind.SHORT);
}
} catch (BufferUnderflowException ex) {
throw new InvalidDataException("Node data is incomplete", ex);
}
}
/**
* Constructs a coordination service node data data transfer object for an
* auto ingest manifest from values provided by the auto ingest system.
* Gets the processing status of the job.
*
* @param manifest The manifest
* @param status The processing status of the manifest.
* @param priority The priority of the manifest.
* @param numberOfCrashes The number of times auto ingest jobs for the
* manifest have crashed during processing.
* @param completedDate The date the auto ingest job for the manifest was
* completed.
* @param errorsOccurred Boolean to determine if errors have occurred.
* @return The processing status.
*/
AutoIngestJobNodeData(Manifest manifest, ProcessingStatus status, int priority, int numberOfCrashes, Date completedDate, boolean errorOccurred) {
this.coordSvcNodeDataWasSet = false;
this.status = status;
this.priority = priority;
this.numberOfCrashes = numberOfCrashes;
this.completedDate = completedDate.getTime();
this.errorsOccurred = errorOccurred;
this.version = NODE_DATA_VERSION;
this.deviceId = manifest.getDeviceId();
this.caseName = manifest.getCaseName();
this.manifestFileDate = manifest.getDateFileCreated().getTime();
this.manifestFilePath = manifest.getFilePath().toString();
this.dataSourcePath = manifest.getDataSourcePath().toString();
AutoIngestJob.ProcessingStatus getProcessingStatus() {
return AutoIngestJob.ProcessingStatus.values()[this.processingStatus];
}
/**
* Indicates whether or not the coordination service node data was set,
* i.e., this object was constructed from raw bytes from the ccordination
* service node for the manifest.
* Sets the processing status of the job.
*
* @return True or false.
* @param processingSatus The processing status.
*/
boolean coordSvcNodeDataWasSet() {
return this.coordSvcNodeDataWasSet;
void setProcessingStatus(AutoIngestJob.ProcessingStatus processingStatus) {
this.processingStatus = processingStatus.ordinal();
}
/**
* Gets the processing status of the manifest
* Gets the priority of the job.
*
* @return The processing status of the manifest.
*/
ProcessingStatus getStatus() {
return this.status;
}
/**
* Sets the processing status of the manifest
*
* @param status The processing status of the manifest.
*/
void setStatus(ProcessingStatus status) {
this.status = status;
}
/**
* Gets the priority of the manifest.
*
* @return The priority of the manifest.
* @return The priority.
*/
int getPriority() {
return this.priority;
}
/**
* Sets the priority of the manifest. A higher number indicates a higheer
* Sets the priority of the job. A higher number indicates a higheer
* priority.
*
* @param priority The priority of the manifest.
* @param priority The priority.
*/
void setPriority(int priority) {
this.priority = priority;
}
/**
* Gets the number of times auto ingest jobs for the manifest have crashed
* during processing.
* Gets the number of times the job has crashed during processing.
*
* @return The number of times auto ingest jobs for the manifest have
* crashed during processing.
* @return The number of crashes.
*/
int getNumberOfCrashes() {
return this.numberOfCrashes;
}
/**
* Sets the number of times auto ingest jobs for the manifest have crashed
* during processing.
* Sets the number of times the job has crashed during processing.
*
* @param numberOfCrashes The number of times auto ingest jobs for the
* manifest have crashed during processing.
* @param numberOfCrashes The number of crashes.
*/
void setNumberOfCrashes(int numberOfCrashes) {
this.numberOfCrashes = numberOfCrashes;
}
/**
* Gets the date the auto ingest job for the manifest was completed.
* Gets the date the job was completed. A completion date equal to the epoch
* (January 1, 1970, 00:00:00 GMT), i.e., Date.getTime() returns 0L,
* indicates the job has not been completed.
*
* @return The date the auto ingest job for the manifest was completed. The
* epoch (January 1, 1970, 00:00:00 GMT) indicates the date is not
* set, i.e., Date.getTime() returns 0L.
* @return The job completion date.
*/
Date getCompletedDate() {
return new Date(this.completedDate);
}
/**
* Sets the date the auto ingest job for the manifest was completed.
* Sets the date the job was completed. A completion date equal to the epoch
* (January 1, 1970, 00:00:00 GMT), i.e., Date.getTime() returns 0L,
* indicates the job has not been completed.
*
* @param completedDate The date the auto ingest job for the manifest was
* completed. Use the epoch (January 1, 1970, 00:00:00
* GMT) to indicate the date is not set, i.e., new
* Date(0L).
* @param completedDate The job completion date.
*/
void setCompletedDate(Date completedDate) {
this.completedDate = completedDate.getTime();
}
/**
* Queries whether or not any errors occurred during the processing of the
* auto ingest job for the manifest.
* Gets whether or not any errors occurred during the processing of the job.
*
* @return True or false.
*/
@ -279,8 +247,7 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Sets whether or not any errors occurred during the processing of the auto
* ingest job for the manifest.
* Sets whether or not any errors occurred during the processing of job.
*
* @param errorsOccurred True or false.
*/
@ -289,25 +256,17 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Get the node data version.
* Gets the node data version number.
*
* @return The node data version.
* @return The version number.
*/
int getVersion() {
return this.version;
}
/**
* Set the node data version.
*
* @param version The node data version.
*/
void setVersion(int version) {
this.version = version;
}
/**
* Get the device ID.
* Gets the device ID of the device associated with the data source for the
* job.
*
* @return The device ID.
*/
@ -316,7 +275,8 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Set the device ID.
* Sets the device ID of the device associated with the data source for the
* job.
*
* @param deviceId The device ID.
*/
@ -325,7 +285,7 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Get the case name.
* Gets the case name.
*
* @return The case name.
*/
@ -334,7 +294,7 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Set the case name.
* Sets the case name.
*
* @param caseName The case name.
*/
@ -342,12 +302,37 @@ final class AutoIngestJobNodeData implements Serializable {
this.caseName = caseName;
}
/**
* Sets the path to the case directory of the case associated with the job.
*
* @param caseDirectoryPath The path to the case directory.
*/
synchronized void setCaseDirectoryPath(Path caseDirectoryPath) {
if (caseDirectoryPath == null) {
this.caseDirectoryPath = "";
} else {
this.caseDirectoryPath = caseDirectoryPath.toString();
}
}
/**
* Gets the path to the case directory of the case associated with the job.
*
* @return The case directory path or null if the case directory has not
* been created yet.
*/
synchronized Path getCaseDirectoryPath() {
if (!caseDirectoryPath.isEmpty()) {
return Paths.get(caseDirectoryPath);
} else {
return null;
}
}
/**
* Gets the date the manifest was created.
*
* @return The date the manifest was created. The epoch (January 1, 1970,
* 00:00:00 GMT) indicates the date is not set, i.e., Date.getTime()
* returns 0L.
* @return The date the manifest was created.
*/
Date getManifestFileDate() {
return new Date(this.manifestFileDate);
@ -356,16 +341,14 @@ final class AutoIngestJobNodeData implements Serializable {
/**
* Sets the date the manifest was created.
*
* @param manifestFileDate The date the manifest was created. Use the epoch
* (January 1, 1970, 00:00:00 GMT) to indicate the
* date is not set, i.e., new Date(0L).
* @param manifestFileDate The date the manifest was created.
*/
void setManifestFileDate(Date manifestFileDate) {
this.manifestFileDate = manifestFileDate.getTime();
}
/**
* Get the manifest file path.
* Gets the manifest file path.
*
* @return The manifest file path.
*/
@ -374,7 +357,7 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Set the manifest file path.
* Sets the manifest file path.
*
* @param manifestFilePath The manifest file path.
*/
@ -387,7 +370,7 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Get the data source path.
* Gets the path of the data source for the job.
*
* @return The data source path.
*/
@ -396,7 +379,7 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Get the file name portion of the data source path.
* Get the file name portion of the path of the data source for the job.
*
* @return The data source file name.
*/
@ -405,7 +388,7 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Set the data source path.
* Sets the path of the data source for the job.
*
* @param dataSourcePath The data source path.
*/
@ -418,16 +401,91 @@ final class AutoIngestJobNodeData implements Serializable {
}
/**
* Gets the node data as raw bytes that can be sent to the coordination
* Get the processing stage of the job.
*
* @return The processing stage.
*/
AutoIngestJob.Stage getProcessingStage() {
return AutoIngestJob.Stage.values()[this.processingStage];
}
/**
* Sets the processing stage job.
*
* @param processingStage The processing stage.
*/
void setProcessingStage(AutoIngestJob.Stage processingStage) {
this.processingStage = (byte) processingStage.ordinal();
}
/**
* Gets the processing stage start date.
*
* @return The processing stage start date.
*/
Date getProcessingStageStartDate() {
return new Date(this.processingStageStartDate);
}
/**
* Sets the processing stage start date.
*
* @param processingStageStartDate The processing stage start date.
*/
void setProcessingStageStartDate(Date processingStageStartDate) {
this.processingStageStartDate = processingStageStartDate.getTime();
}
/**
* Get the processing stage details.
*
* @return A processing stage details object.
*/
AutoIngestJob.StageDetails getProcessingStageDetails() {
return new AutoIngestJob.StageDetails(this.processingStageDetailsDescription, new Date(this.processingStageDetailsStartDate));
}
/**
* Sets the details of the current processing stage.
*
* @param stageDetails A stage details object.
*/
void setProcessingStageDetails(AutoIngestJob.StageDetails stageDetails) {
this.processingStageDetailsDescription = stageDetails.getDescription();
this.processingStageDetailsStartDate = stageDetails.getStartDate().getTime();
}
/**
* Gets the processing host name, may be the empty string.
*
* @return The processing host. The empty string if the job is not currently
* being processed.
*/
String getProcessingHostName() {
return this.processingHostName;
}
/**
* Sets the processing host name. May be the empty string.
*
* @param processingHost The processing host name. The empty string if the
* job is not currently being processed.
*/
void setProcessingHostName(String processingHost) {
this.processingHostName = processingHost;
}
/**
* Gets the node data as a byte array that can be sent to the coordination
* service.
*
* @return The manifest node data as a byte array.
* @return The node data as a byte array.
*/
byte[] toArray() {
ByteBuffer buffer = ByteBuffer.allocate(MAX_POSSIBLE_NODE_DATA_SIZE);
// Write data (compatible with version 0)
buffer.putInt(this.status.ordinal());
buffer.putInt(this.processingStatus);
buffer.putInt(this.priority);
buffer.putInt(this.numberOfCrashes);
buffer.putLong(this.completedDate);
@ -440,13 +498,15 @@ final class AutoIngestJobNodeData implements Serializable {
// Write data
putStringIntoBuffer(deviceId, buffer, TypeKind.BYTE);
putStringIntoBuffer(caseName, buffer, TypeKind.BYTE);
//DLG: putStringIntoBuffer(caseDirectoryPath, buffer, TypeKind.SHORT);
putStringIntoBuffer(caseDirectoryPath, buffer, TypeKind.SHORT);
buffer.putLong(this.manifestFileDate);
putStringIntoBuffer(manifestFilePath, buffer, TypeKind.SHORT);
putStringIntoBuffer(dataSourcePath, buffer, TypeKind.SHORT);
//DLG: putStringIntoBuffer(processingStage, buffer, TypeKind.BYTE);
//DLG: buffer.putLong(this.processingStageStartDate);
//DLG: putStringIntoBuffer(processingHost, buffer, TypeKind.SHORT);
buffer.put(this.processingStage);
buffer.putLong(this.processingStageStartDate);
putStringIntoBuffer(this.processingStageDetailsDescription, buffer, TypeKind.BYTE);
buffer.putLong(this.processingStageDetailsStartDate);
putStringIntoBuffer(processingHostName, buffer, TypeKind.SHORT);
}
// Prepare the array
@ -457,6 +517,7 @@ final class AutoIngestJobNodeData implements Serializable {
return array;
}
// DGL: Document what is going on here and how the max buffer sie constant is calculated.
private String getStringFromBuffer(ByteBuffer buffer, TypeKind lengthType) {
int length = 0;
String output = "";
@ -479,6 +540,7 @@ final class AutoIngestJobNodeData implements Serializable {
return output;
}
// DGL: Document what is going on here and how the max buffer sie constant is calculated.
private void putStringIntoBuffer(String stringValue, ByteBuffer buffer, TypeKind lengthType) {
switch (lengthType) {
case BYTE:
@ -492,14 +554,17 @@ final class AutoIngestJobNodeData implements Serializable {
buffer.put(stringValue.getBytes());
}
/**
* Processing status for the auto ingest job for the manifest.
*/
enum ProcessingStatus {
PENDING,
PROCESSING,
COMPLETED,
DELETED
final static class InvalidDataException extends Exception {
private static final long serialVersionUID = 1L;
private InvalidDataException(String message) {
super(message);
}
private InvalidDataException(String message, Throwable cause) {
super(message, cause);
}
}
}

46
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobNodeDataException.java
View File

@ -1,46 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.experimental.autoingest;
/**
* Exception thrown when a manifest node contains incompatible data.
*/
public class AutoIngestJobNodeDataException extends Exception {
/**
* Constructs an exception thrown when a manifest node contains incompatible
* data.
*
* @param message An error message.
*/
public AutoIngestJobNodeDataException(String message) {
super(message);
}
/**
* Constructs an exception thrown when a manifest node contains incompatible
* data.
*
* @param message An error message.
* @param cause An exception that caused this exception to be thrown.
*/
public AutoIngestJobNodeDataException(String message, Throwable cause) {
super(message, cause);
}
}

693
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
View File

File diff suppressed because it is too large Load Diff

67
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java
View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.experimental.autoingest;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.awt.Cursor;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.nio.file.Path;
@ -33,12 +34,14 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import javax.annotation.concurrent.GuardedBy;
import org.openide.util.Exceptions;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.events.AutopsyEventException;
import org.sleuthkit.autopsy.events.AutopsyEventPublisher;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.ProcessingStatus;
/**
* An auto ingest monitor responsible for monitoring and reporting the
@ -140,8 +143,10 @@ public final class AutoIngestMonitor extends Observable implements PropertyChang
*/
private void handleJobStartedEvent(AutoIngestJobStartedEvent event) {
synchronized (jobsLock) {
// DLG: Remove job from pending queue, if present
// DLG: Add job to running jobs list
// DLG: TEST! Remove job from pending queue, if present
// DLG: TEST! Add job to running jobs list
jobsSnapshot.removePendingJob(event.getJob());
jobsSnapshot.addOrReplaceRunningJob(event.getJob());
setChanged();
notifyObservers(jobsSnapshot);
}
@ -154,7 +159,8 @@ public final class AutoIngestMonitor extends Observable implements PropertyChang
*/
private void handleJobStatusEvent(AutoIngestJobStatusEvent event) {
synchronized (jobsLock) {
// DLG: Replace job in running list with job from event
// DLG: TEST! Replace job in running list with job from event
jobsSnapshot.addOrReplaceRunningJob(event.getJob());
setChanged();
notifyObservers(jobsSnapshot);
}
@ -167,8 +173,10 @@ public final class AutoIngestMonitor extends Observable implements PropertyChang
*/
private void handleJobCompletedEvent(AutoIngestJobCompletedEvent event) {
synchronized (jobsLock) {
// DLG: Remove job from event from running list, if present
// DLG: Add job to completed list
// DLG: TEST! Remove job from event from running list, if present
// DLG: TEST! Add job to completed list
jobsSnapshot.removeRunningJob(event.getJob());
jobsSnapshot.addOrReplaceCompletedJob(event.getJob());
setChanged();
notifyObservers(jobsSnapshot);
}
@ -180,12 +188,7 @@ public final class AutoIngestMonitor extends Observable implements PropertyChang
* @param event A job/case prioritization event.
*/
private void handleCasePrioritizationEvent(AutoIngestCasePrioritizedEvent event) {
synchronized (jobsLock) {
// DLG: Replace job in pending queue with job from event
// DLG: See 'bnPrioritizeCaseActionPerformed(ActionEvent)' in the AutoIngestControlPanel class!!!
setChanged();
notifyObservers(jobsSnapshot);
}
coordSvcQueryExecutor.submit(new CoordinationServiceQueryTask());
}
/**
@ -236,9 +239,31 @@ public final class AutoIngestMonitor extends Observable implements PropertyChang
JobsSnapshot newJobsSnapshot = new JobsSnapshot();
List<String> nodeList = coordinationService.getNodeList(CoordinationService.CategoryNode.MANIFESTS);
for (String node : nodeList) {
// DLG: Do not need a lock here
// DLG: Get the node data and construct a AutoIngestJobNodeData object (rename AutoIngestJobNodeData => AutoIngestJobData)
// DLG: Construct an AutoIngestJob object from the AutoIngestJobNodeData object, need new AutoIngestJob constructor
try {
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, node));
AutoIngestJob job = new AutoIngestJob(nodeData);
ProcessingStatus processingStatus = nodeData.getProcessingStatus();
switch (processingStatus) {
case PENDING:
newJobsSnapshot.addOrReplacePendingJob(job);
break;
case PROCESSING:
newJobsSnapshot.addOrReplaceRunningJob(job);
break;
case COMPLETED:
newJobsSnapshot.addOrReplaceCompletedJob(job);
break;
case DELETED:
break;
default:
LOGGER.log(Level.SEVERE, "Unknown AutoIngestJobData.ProcessingStatus");
break;
}
} catch (InterruptedException ex) {
LOGGER.log(Level.SEVERE, String.format("Unexpected interrupt while retrieving coordination service node data for '%s'", node), ex);
} catch (AutoIngestJobNodeData.InvalidDataException ex) {
LOGGER.log(Level.SEVERE, String.format("Unable to use node data for '%s'", node), ex);
}
}
return newJobsSnapshot;
} catch (CoordinationServiceException ex) {
@ -261,10 +286,10 @@ public final class AutoIngestMonitor extends Observable implements PropertyChang
* the pending jobs queue.
*/
for (AutoIngestJob job : jobsSnapshot.getPendingJobs()) {
if (job.getNodeData().getPriority() > highestPriority) {
highestPriority = job.getNodeData().getPriority();
if (job.getPriority() > highestPriority) {
highestPriority = job.getPriority();
}
if (job.getNodeData().getManifestFilePath().equals(manifestFilePath)) {
if (job.getManifest().getFilePath().equals(manifestFilePath)) {
prioritizedJob = job;
}
}
@ -275,22 +300,22 @@ public final class AutoIngestMonitor extends Observable implements PropertyChang
*/
if (null != prioritizedJob) {
++highestPriority;
String manifestNodePath = prioritizedJob.getNodeData().getManifestFilePath().toString();
String manifestNodePath = prioritizedJob.getManifest().getFilePath().toString();
try {
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestNodePath));
nodeData.setPriority(highestPriority);
coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestNodePath, nodeData.toArray());
} catch (AutoIngestJobNodeDataException | CoordinationServiceException | InterruptedException ex) {
} catch (AutoIngestJobNodeData.InvalidDataException | CoordinationServiceException | InterruptedException ex) {
throw new AutoIngestMonitorException("Error bumping priority for job " + prioritizedJob.toString(), ex);
}
prioritizedJob.getNodeData().setPriority(highestPriority);
prioritizedJob.setPriority(highestPriority);
}
/*
* Publish a prioritization event.
*/
if (null != prioritizedJob) {
final String caseName = prioritizedJob.getNodeData().getCaseName();
final String caseName = prioritizedJob.getManifest().getCaseName();
new Thread(() -> {
eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName));
}).start();

112
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutopsyManifestFileParser.java
View File

@ -18,11 +18,11 @@
*/
package org.sleuthkit.autopsy.experimental.autoingest;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Date;
import java.util.HashMap;
import javax.annotation.concurrent.Immutable;
import javax.xml.parsers.DocumentBuilder;
@ -42,24 +42,11 @@ import org.xml.sax.SAXException;
public final class AutopsyManifestFileParser implements ManifestFileParser {
private static final String MANIFEST_FILE_NAME_SIGNATURE = "_MANIFEST.XML";
private static final String NMEC_MANIFEST_ELEM_TAG_NAME = "NMEC_Manifest";
private static final String MANIFEST_ELEM_TAG_NAME = "Manifest";
private static final String CASE_NAME_XPATH = "/Collection/Name/text()";
private static final String DEVICE_ID_XPATH = "/Collection/Image/ID/text()";
private static final String IMAGE_NAME_XPATH = "/Collection/Image/Name/text()";
private static final String IMAGE_FULL_NAME_XPATH = "/Collection/Image/FullName/text()";
private static final String IMAGE_RELATIVE_PATH_XPATH = "/Collection/Image/RelativePath/text()";
private static final String ROOT_ELEM_TAG_NAME = "AutopsyManifest";
private static final String CASE_NAME_XPATH = "/AutopsyManifest/CaseName/text()";
private static final String DEVICE_ID_XPATH = "/AutopsyManifest/DeviceId/text()";
private static final String DATA_SOURCE_NAME_XPATH = "/AutopsyManifest/DataSource/text()";
private String actualRootElementTag = "";
/**
* Determine whether the given file is a supported manifest file.
*
* @param filePath
*
* @return true if this is a supported manifest file, otherwise false
*/
@Override
public boolean fileIsManifest(Path filePath) {
boolean fileIsManifest = false;
@ -68,9 +55,7 @@ public final class AutopsyManifestFileParser implements ManifestFileParser {
if (fileName.toString().toUpperCase().endsWith(MANIFEST_FILE_NAME_SIGNATURE)) {
Document doc = this.createManifestDOM(filePath);
Element docElement = doc.getDocumentElement();
actualRootElementTag = docElement.getTagName();
fileIsManifest = actualRootElementTag.equals(MANIFEST_ELEM_TAG_NAME) ||
actualRootElementTag.equals(NMEC_MANIFEST_ELEM_TAG_NAME);
fileIsManifest = docElement.getTagName().equals(ROOT_ELEM_TAG_NAME);
}
} catch (Exception unused) {
fileIsManifest = false;
@ -78,95 +63,30 @@ public final class AutopsyManifestFileParser implements ManifestFileParser {
return fileIsManifest;
}
/**
* Parse the given manifest file and create a Manifest object.
*
* @param filePath Fully qualified path to manifest file
*
* @return A Manifest object representing the parsed manifest file.
*
* @throws ManifestFileParserException
*/
@Override
public Manifest parse(Path filePath) throws ManifestFileParserException {
try {
BasicFileAttributes attrs = Files.readAttributes(filePath, BasicFileAttributes.class);
Date dateFileCreated = new Date(attrs.creationTime().toMillis());
Document doc = this.createManifestDOM(filePath);
XPath xpath = XPathFactory.newInstance().newXPath();
XPathExpression expr = xpath.compile(constructXPathExpression(CASE_NAME_XPATH));
XPathExpression expr = xpath.compile(CASE_NAME_XPATH);
String caseName = (String) expr.evaluate(doc, XPathConstants.STRING);
expr = xpath.compile(constructXPathExpression(DEVICE_ID_XPATH));
expr = xpath.compile(DEVICE_ID_XPATH);
String deviceId = (String) expr.evaluate(doc, XPathConstants.STRING);
Path dataSourcePath = determineDataSourcePath(filePath, doc);
return new Manifest(filePath, caseName, deviceId, dataSourcePath, new HashMap<>());
expr = xpath.compile(DATA_SOURCE_NAME_XPATH);
String dataSourceName = (String) expr.evaluate(doc, XPathConstants.STRING);
Path dataSourcePath = filePath.getParent().resolve(dataSourceName);
return new Manifest(filePath, dateFileCreated, caseName, deviceId, dataSourcePath, new HashMap<>());
} catch (Exception ex) {
throw new ManifestFileParserException(String.format("Error parsing manifest %s", filePath), ex);
}
}
/**
* Create a new DOM document object for the given manifest file.
*
* @param manifestFilePath Fully qualified path to manifest file.
*
* @return DOM document object
*
* @throws ParserConfigurationException
* @throws SAXException
* @throws IOException
*/
private Document createManifestDOM(Path manifestFilePath) throws ParserConfigurationException, SAXException, IOException {
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
return docBuilder.parse(manifestFilePath.toFile());
}
/**
* Creates an XPath expression string relative to the actual root
* element of the manifest for the given path.
*
* @param path
* @return XPath expression string.
*/
private String constructXPathExpression(String path) {
return "/" + actualRootElementTag + path;
}
/**
* Attempt to find a valid (existing) data source for the manifest file.
*
* @param manifestFilePath Fully qualified path to manifest file.
* @param doc DOM document object for the manifest file.
* @return Path to an existing data source.
* @throws ManifestFileParserException if an error occurred while parsing manifest file.
*/
private Path determineDataSourcePath(Path manifestFilePath, Document doc) throws ManifestFileParserException {
String dataSourcePath = "";
try {
for (String element : Arrays.asList(IMAGE_NAME_XPATH, IMAGE_FULL_NAME_XPATH, IMAGE_RELATIVE_PATH_XPATH)) {
XPath xpath = XPathFactory.newInstance().newXPath();
XPathExpression expr = xpath.compile(constructXPathExpression(element));
String fileName = (String) expr.evaluate(doc, XPathConstants.STRING);
if (fileName.contains("\\")) {
fileName = fileName.substring(fileName.lastIndexOf("\\") + 1);
}
try {
dataSourcePath = manifestFilePath.getParent().resolve(fileName).toString();
} catch (Exception ignore) {
// NOTE: exceptions can be thrown by resolve() method based on contents of the manifest file.
// For example if file name is "test .txt" and in one of the path fields they only enter "test "
// i.e. the file name without extension.
// We should continue on to the next XML path field
continue;
}
if (new File(dataSourcePath).exists()) {
// found the data source
return Paths.get(dataSourcePath);
}
// keep trying other XML fields
}
return Paths.get(dataSourcePath);
} catch (Exception ex) {
throw new ManifestFileParserException(String.format("Error parsing manifest %s", manifestFilePath), ex);
}
}
}

2
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties
View File

@ -27,7 +27,6 @@ AutoIngestDashboard.DeletionFailed=Deletion failed for job
AutoIngestDashboard.ShowLogFailed.Title=Unable to display case log
AutoIngestDashboard.ShowLogFailed.Message=Case log file does not exist
AutoIngestDashboard.bnPrioritizeCase.toolTipText=Move all images associated with a case to top of Pending queue.
AutoIngestDashboard.bnPrioritizeCase.text=Prioritize Case
AutoIngestDashboard.ExitConsequences=This will cancel any currently running job on this host. Exiting while a job is running potentially leaves the case in an inconsistent or corrupted state.
AutoIngestDashboard.ExitingStatus=Exiting...
AutoIngestDashboard.OK=OK
@ -275,3 +274,4 @@ AutoIngestDashboard.prioritizeButton.toolTipText=Prioritizes the selected job
AutoIngestDashboard.prioritizeButton.text=&Prioritize
AutoIngestDashboard.refreshButton.toolTipText=Refresh displayed tables
AutoIngestDashboard.refreshButton.text=&Refresh
AutoIngestDashboard.jButton1.text=jButton1

10
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Manifest.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2015 Basis Technology Corp.
* Copyright 2011-2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -18,12 +18,9 @@
*/
package org.sleuthkit.autopsy.experimental.autoingest;
import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
@ -40,10 +37,9 @@ public final class Manifest implements Serializable {
private final String dataSourcePath;
private final Map<String, String> manifestProperties;
public Manifest(Path manifestFilePath, String caseName, String deviceId, Path dataSourcePath, Map<String, String> manifestProperties) throws IOException {
public Manifest(Path manifestFilePath, Date dateFileCreated, String caseName, String deviceId, Path dataSourcePath, Map<String, String> manifestProperties) {
this.filePath = manifestFilePath.toString();
BasicFileAttributes attrs = Files.readAttributes(manifestFilePath, BasicFileAttributes.class);
this.dateFileCreated = new Date(attrs.creationTime().toMillis());
this.dateFileCreated = dateFileCreated;
this.caseName = caseName;
this.deviceId = deviceId;
if (dataSourcePath != null) {

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractedContentPanel.java
View File

@ -637,7 +637,7 @@ class ExtractedContentPanel extends javax.swing.JPanel {
}
@Override
@NbBundle.Messages({"# 0 - Content name",
@NbBundle.Messages({"# {0} - Content name",
"ExtractedContentPanel.SetMarkup.progress.loading=Loading text for {0}"})
protected String doInBackground() throws Exception {
progress = ProgressHandle.createHandle(Bundle.ExtractedContentPanel_SetMarkup_progress_loading(contentName));

3
docs/doxygen-user/central_repo.dox
View File

@ -68,6 +68,9 @@ and added to the list of Interesting Items.
\image html central_repo_manage_tags.png
If a case is open, checking the Implies Known Bad checkbox will give you the option to add the known bad status to anything in the current case
that has already been tagged. For example, if you create a tag named "Alpha", tag a few items and then go into Manage Tags and check the box for the Alpha tag, you can optionally choose to have the status for those tagged items changed in the Central Repository. The effect is the same as if you had checked the box in Manage Tags before tagging the items. Note that data from any previous cases will not be changed.
By default there is a tag called "Evidence" as the only tag associated with this module. To associate one or more tag(s) with this module, check the Correlate box next to the tag
name(s) and click OK. To create additional tags, use the Tags options panel.