mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-15 09:17:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Complete improved ingest framework

Browse Source
This commit is contained in:
Richard Cordovano 2014-05-02 15:03:46 -04:00
parent 07e9956ee0
commit c900f0f95a
12 changed files with 262 additions and 661 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.form
View File

@ -188,15 +188,12 @@
<Component class="javax.swing.JComboBox" name="numberOfFileIngestThreadsComboBox"> <Component class="javax.swing.JComboBox" name="numberOfFileIngestThreadsComboBox">
<Properties> <Properties>
<Property name="model" type="javax.swing.ComboBoxModel" editor="org.netbeans.modules.form.editors2.ComboBoxModelEditor"> <Property name="model" type="javax.swing.ComboBoxModel" editor="org.netbeans.modules.form.editors2.ComboBoxModelEditor">
<StringArray count="4"> <StringArray count="0"/>
<StringItem index="0" value="1"/>
<StringItem index="1" value="2"/>
<StringItem index="2" value="3"/>
<StringItem index="3" value="4"/>
</StringArray>
</Property> </Property>
<Property name="selectedIndex" type="int" value="1"/>
</Properties> </Properties>
<AuxValues>
<AuxValue name="JavaCodeGenerator_TypeParameters" type="java.lang.String" value="&lt;Integer&gt;"/>
</AuxValues>
</Component> </Component>
</SubComponents> </SubComponents>
</Form> </Form>

60
Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.java
View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.corecomponents; package org.sleuthkit.autopsy.corecomponents;
import java.util.prefs.Preferences; import java.util.prefs.Preferences;
import javax.swing.DefaultComboBoxModel;
import org.openide.util.NbPreferences; import org.openide.util.NbPreferences;
import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestManager;
@ -33,10 +34,35 @@ final class GeneralPanel extends javax.swing.JPanel {
GeneralPanel(GeneralOptionsPanelController controller) { GeneralPanel(GeneralOptionsPanelController controller) {
initComponents(); initComponents();
numberOfFileIngestThreadsComboBox.setModel(new DefaultComboBoxModel<>(new Integer[]{1, 2, 4, 8, 16}));
ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected()); ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected());
// TODO listen to changes in form fields and call controller.changed() // TODO listen to changes in form fields and call controller.changed()
} }
void load() {
boolean keepPreferredViewer = prefs.getBoolean(KEEP_PREFERRED_VIEWER, false);
keepCurrentViewerRB.setSelected(keepPreferredViewer);
useBestViewerRB.setSelected(!keepPreferredViewer);
boolean useLocalTime = prefs.getBoolean(USE_LOCAL_TIME, true);
useLocalTimeRB.setSelected(useLocalTime);
useGMTTimeRB.setSelected(!useLocalTime);
dataSourcesHideKnownCB.setSelected(prefs.getBoolean(DS_HIDE_KNOWN, false));
viewsHideKnownCB.setSelected(prefs.getBoolean(VIEWS_HIDE_KNOWN, true));
numberOfFileIngestThreadsComboBox.setSelectedItem(IngestManager.getNumberOfFileIngestThreads());
}
void store() {
prefs.putBoolean(KEEP_PREFERRED_VIEWER, keepCurrentViewerRB.isSelected());
prefs.putBoolean(USE_LOCAL_TIME, useLocalTimeRB.isSelected());
prefs.putBoolean(DS_HIDE_KNOWN, dataSourcesHideKnownCB.isSelected());
prefs.putBoolean(VIEWS_HIDE_KNOWN, viewsHideKnownCB.isSelected());
IngestManager.setNumberOfFileIngestThreads((Integer) numberOfFileIngestThreadsComboBox.getSelectedItem());
}
boolean valid() {
return true;
}
/** /**
* This method is called from within the constructor to initialize the form. * This method is called from within the constructor to initialize the form.
* WARNING: Do NOT modify this code. The content of this method is always * WARNING: Do NOT modify this code. The content of this method is always
@ -57,7 +83,7 @@ final class GeneralPanel extends javax.swing.JPanel {
dataSourcesHideKnownCB = new javax.swing.JCheckBox(); dataSourcesHideKnownCB = new javax.swing.JCheckBox();
viewsHideKnownCB = new javax.swing.JCheckBox(); viewsHideKnownCB = new javax.swing.JCheckBox();
jLabel4 = new javax.swing.JLabel(); jLabel4 = new javax.swing.JLabel();
numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox(); numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox<Integer>();
buttonGroup1.add(useBestViewerRB); buttonGroup1.add(useBestViewerRB);
useBestViewerRB.setSelected(true); useBestViewerRB.setSelected(true);
@ -97,9 +123,6 @@ final class GeneralPanel extends javax.swing.JPanel {
org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(GeneralPanel.class, "GeneralPanel.jLabel4.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(GeneralPanel.class, "GeneralPanel.jLabel4.text")); // NOI18N
numberOfFileIngestThreadsComboBox.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "1", "2", "3", "4" }));
numberOfFileIngestThreadsComboBox.setSelectedIndex(1);
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout); this.setLayout(layout);
layout.setHorizontalGroup( layout.setHorizontalGroup(
@ -163,33 +186,8 @@ final class GeneralPanel extends javax.swing.JPanel {
}//GEN-LAST:event_useBestViewerRBActionPerformed }//GEN-LAST:event_useBestViewerRBActionPerformed
private void useGMTTimeRBActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useGMTTimeRBActionPerformed private void useGMTTimeRBActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useGMTTimeRBActionPerformed
ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected()); ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected());
}//GEN-LAST:event_useGMTTimeRBActionPerformed }//GEN-LAST:event_useGMTTimeRBActionPerformed
void load() {
boolean keepPreferredViewer = prefs.getBoolean(KEEP_PREFERRED_VIEWER, false);
keepCurrentViewerRB.setSelected(keepPreferredViewer);
useBestViewerRB.setSelected(!keepPreferredViewer);
boolean useLocalTime = prefs.getBoolean(USE_LOCAL_TIME, true);
useLocalTimeRB.setSelected(useLocalTime);
useGMTTimeRB.setSelected(!useLocalTime);
dataSourcesHideKnownCB.setSelected(prefs.getBoolean(DS_HIDE_KNOWN, false));
viewsHideKnownCB.setSelected(prefs.getBoolean(VIEWS_HIDE_KNOWN, true));
numberOfFileIngestThreadsComboBox.setSelectedItem(IngestManager.getInstance().getNumberOfFileIngestThreads());
}
void store() {
prefs.putBoolean(KEEP_PREFERRED_VIEWER, keepCurrentViewerRB.isSelected());
prefs.putBoolean(USE_LOCAL_TIME, useLocalTimeRB.isSelected());
prefs.putBoolean(DS_HIDE_KNOWN, dataSourcesHideKnownCB.isSelected());
prefs.putBoolean(VIEWS_HIDE_KNOWN, viewsHideKnownCB.isSelected());
IngestManager.getInstance().setNumberOfFileIngestThreads(Integer.valueOf(numberOfFileIngestThreadsComboBox.getSelectedItem().toString()));
}
boolean valid() {
// TODO check whether form is consistent and complete
return true;
}
// Variables declaration - do not modify//GEN-BEGIN:variables // Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.ButtonGroup buttonGroup1; private javax.swing.ButtonGroup buttonGroup1;
private javax.swing.ButtonGroup buttonGroup3; private javax.swing.ButtonGroup buttonGroup3;
@ -199,7 +197,7 @@ final class GeneralPanel extends javax.swing.JPanel {
private javax.swing.JLabel jLabel3; private javax.swing.JLabel jLabel3;
private javax.swing.JLabel jLabel4; private javax.swing.JLabel jLabel4;
private javax.swing.JRadioButton keepCurrentViewerRB; private javax.swing.JRadioButton keepCurrentViewerRB;
private javax.swing.JComboBox numberOfFileIngestThreadsComboBox; private javax.swing.JComboBox<Integer> numberOfFileIngestThreadsComboBox;
private javax.swing.JRadioButton useBestViewerRB; private javax.swing.JRadioButton useBestViewerRB;
private javax.swing.JRadioButton useGMTTimeRB; private javax.swing.JRadioButton useGMTTimeRB;
private javax.swing.JRadioButton useLocalTimeRB; private javax.swing.JRadioButton useLocalTimeRB;

15
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java
View File

@ -1,7 +1,7 @@
/* /*
* Autopsy Forensic Browser * Autopsy Forensic Browser
* *
* Copyright 2014 Basis Technology Corp. * Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org * Contact: carrier <at> sleuthkit <dot> org
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
@ -20,26 +20,21 @@ package org.sleuthkit.autopsy.ingest;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
final class DataSourceIngestTask implements IngestTask { final class DataSourceIngestTask extends IngestTask {
private final IngestJob ingestJob;
private final Content dataSource; private final Content dataSource;
DataSourceIngestTask(IngestJob ingestJob, Content dataSource) { DataSourceIngestTask(IngestJob ingestJob, Content dataSource) {
this.ingestJob = ingestJob; super(ingestJob);
this.dataSource = dataSource; this.dataSource = dataSource;
} }
IngestJob getIngestJob() {
return ingestJob;
}
Content getDataSource() { Content getDataSource() {
return dataSource; return dataSource;
} }
@Override @Override
public void execute() throws InterruptedException { void execute() throws InterruptedException {
ingestJob.process(dataSource); getIngestJob().process(dataSource);
} }
} }

56
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java
View File

@ -1,56 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.concurrent.LinkedBlockingQueue;
//final class DataSourceIngestTaskScheduler implements IngestTaskQueue {
//
// private static DataSourceIngestTaskScheduler instance = new DataSourceIngestTaskScheduler();
// private final LinkedBlockingQueue<DataSourceIngestTask> tasks = new LinkedBlockingQueue<>();
//
// static DataSourceIngestTaskScheduler getInstance() {
// return instance;
// }
//
// private DataSourceIngestTaskScheduler() {
// }
//
// void addTask(DataSourceIngestTask task) {
// // If the thread executing this code is interrupted, it is because the
// // number of ingest threads has been decreased while ingest jobs are
// // running. This thread will exit in an orderly fashion, but the task
// // still needs to be enqueued rather than lost.
// while (true) {
// try {
// tasks.put(task);
// break;
// } catch (InterruptedException ex) {
// // Reset the interrupted status of the thread so the orderly
// // exit can occur in the intended place.
// Thread.currentThread().interrupt();
// }
// }
// }
//
// @Override
// public DataSourceIngestTask getNextTask() throws InterruptedException {
// return tasks.take();
// }
//}

23
Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java
View File

@ -21,27 +21,22 @@ package org.sleuthkit.autopsy.ingest;
import java.util.Objects; import java.util.Objects;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
final class FileIngestTask implements IngestTask { final class FileIngestTask extends IngestTask {
private final IngestJob ingestJob;
private final AbstractFile file; private final AbstractFile file;
FileIngestTask(IngestJob task, AbstractFile file) { FileIngestTask(IngestJob job, AbstractFile file) {
this.ingestJob = task; super(job);
this.file = file; this.file = file;
} }
public IngestJob getIngestJob() { // RJCTODO: Maybe add to interface AbstractFile getFile() {
return ingestJob;
}
public AbstractFile getFile() {
return file; return file;
} }
@Override @Override
public void execute() throws InterruptedException { void execute() throws InterruptedException {
ingestJob.process(file); getIngestJob().process(file);
} }
@Override @Override
@ -53,7 +48,9 @@ final class FileIngestTask implements IngestTask {
return false; return false;
} }
FileIngestTask other = (FileIngestTask) obj; FileIngestTask other = (FileIngestTask) obj;
if (this.ingestJob != other.ingestJob && (this.ingestJob == null || !this.ingestJob.equals(other.ingestJob))) { IngestJob job = getIngestJob();
IngestJob otherJob = other.getIngestJob();
if (job != otherJob && (job == null || !job.equals(otherJob))) {
return false; return false;
} }
if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) {
@ -65,7 +62,7 @@ final class FileIngestTask implements IngestTask {
@Override @Override
public int hashCode() { public int hashCode() {
int hash = 5; int hash = 5;
hash = 47 * hash + Objects.hashCode(this.ingestJob); hash = 47 * hash + Objects.hashCode(getIngestJob());
hash = 47 * hash + Objects.hashCode(this.file); hash = 47 * hash + Objects.hashCode(this.file);
return hash; return hash;
} }

393
Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java
View File

@ -1,393 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.TreeSet;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.VirtualDirectory;
//final class FileIngestTaskScheduler implements IngestTaskQueue {
//
// private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName());
// private static FileIngestTaskScheduler instance = new FileIngestTaskScheduler();
// private final TreeSet<FileIngestTask> rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator());
// private final List<FileIngestTask> directoryTasks = new ArrayList<>();
// private final LinkedBlockingQueue<FileIngestTask> fileTasks = new LinkedBlockingQueue<>();
// private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
//
// static FileIngestTaskScheduler getInstance() {
// return instance;
// }
//
// private FileIngestTaskScheduler() {
// }
//
// synchronized void addTasks(IngestJob job, Content dataSource) throws InterruptedException {
// Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
// List<AbstractFile> firstLevelFiles = new ArrayList<>();
// if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
// // The data source is file.
// firstLevelFiles.add((AbstractFile) dataSource);
// } else {
// for (AbstractFile root : rootObjects) {
// List<Content> children;
// try {
// children = root.getChildren();
// if (children.isEmpty()) {
// //add the root itself, could be unalloc file, child of volume or image
// firstLevelFiles.add(root);
// } else {
// //root for fs root dir, schedule children dirs/files
// for (Content child : children) {
// if (child instanceof AbstractFile) {
// firstLevelFiles.add((AbstractFile) child);
// }
// }
// }
// } catch (TskCoreException ex) {
// logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS
// }
// }
// }
// for (AbstractFile firstLevelFile : firstLevelFiles) {
// FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile);
// if (shouldEnqueueTask(fileTask)) {
// rootDirectoryTasks.add(fileTask);
// fileTask.getIngestJob().notifyTaskAdded();
// }
// }
//
// // Reshuffle/update the dir and file level queues if needed
// updateQueues();
// }
//
// synchronized void addTask(FileIngestTask task) {
// if (shouldEnqueueTask(task)) {
// addTaskToFileQueue(task, true);
// }
// }
//
// @Override
// public FileIngestTask getNextTask() throws InterruptedException {
// FileIngestTask task = fileTasks.take();
// updateQueues();
// return task;
// }
//
// private void updateQueues() throws InterruptedException {
// // we loop because we could have a directory that has all files
// // that do not get enqueued
// while (true) {
// // There are files in the queue, we're done
// if (fileTasks.isEmpty() == false) {
// return;
// }
// // fill in the directory queue if it is empty.
// if (this.directoryTasks.isEmpty()) {
// // bail out if root is also empty -- we are done
// if (rootDirectoryTasks.isEmpty()) {
// return;
// }
// FileIngestTask rootTask = rootDirectoryTasks.pollFirst();
// directoryTasks.add(rootTask);
// }
// //pop and push AbstractFile directory children if any
// //add the popped and its leaf children onto cur file list
// FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1);
// final AbstractFile parentFile = parentTask.getFile();
// // add itself to the file list
// if (shouldEnqueueTask(parentTask)) {
// addTaskToFileQueue(parentTask, false);
// }
// // add its children to the file and directory lists
// try {
// List<Content> children = parentFile.getChildren();
// for (Content c : children) {
// if (c instanceof AbstractFile) {
// AbstractFile childFile = (AbstractFile) c;
// FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile);
// if (childFile.hasChildren()) {
// directoryTasks.add(childTask);
// childTask.getIngestJob().notifyTaskAdded();
// } else if (shouldEnqueueTask(childTask)) {
// addTaskToFileQueue(childTask, true);
// }
// }
// }
// } catch (TskCoreException ex) {
// logger.log(Level.SEVERE, "Could not get children of file and update file queues: " + parentFile.getName(), ex);
// }
// }
// }
//
// private void addTaskToFileQueue(FileIngestTask task, boolean isNewTask) {
// if (isNewTask) {
// // The capacity of the file tasks queue is not bounded, so the call
// // to put() should not block except for normal synchronized access.
// // Still, notify the job that the task has been added first so that
// // the take() of the task cannot occur before the notification.
// task.getIngestJob().notifyTaskAdded();
// }
// // If the thread executing this code is ever interrupted, it is
// // because the number of ingest threads has been decreased while
// // ingest jobs are running. This thread will exit in an orderly fashion,
// // but the task still needs to be enqueued rather than lost.
// while (true) {
// try {
// fileTasks.put(task);
// break;
// } catch (InterruptedException ex) {
// // Reset the interrupted status of the thread so the orderly
// // exit can occur in the intended place.
// Thread.currentThread().interrupt();
// }
// }
// }
//
// /**
// * Check if the file is a special file that we should skip
// *
// * @param processTask a task whose file to check if should be queued of
// * skipped
// * @return true if should be enqueued, false otherwise
// */
// private static boolean shouldEnqueueTask(final FileIngestTask processTask) {
// final AbstractFile aFile = processTask.getFile();
// //if it's unalloc file, skip if so scheduled
// if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) {
// return false;
// }
// String fileName = aFile.getName();
// if (fileName.equals(".") || fileName.equals("..")) {
// return false;
// } else if (aFile instanceof org.sleuthkit.datamodel.File) {
// final org.sleuthkit.datamodel.File f = (File) aFile;
// //skip files in root dir, starting with $, containing : (not default attributes)
// //with meta address < 32, i.e. some special large NTFS and FAT files
// FileSystem fs = null;
// try {
// fs = f.getFileSystem();
// } catch (TskCoreException ex) {
// logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS
// }
// TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP;
// if (fs != null) {
// fsType = fs.getFsType();
// }
// if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) {
// //not fat or ntfs, accept all files
// return true;
// }
// boolean isInRootDir = false;
// try {
// isInRootDir = f.getParentDirectory().isRoot();
// } catch (TskCoreException ex) {
// logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS
// }
// if (isInRootDir && f.getMetaAddr() < 32) {
// String name = f.getName();
// if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) {
// return false;
// }
// } else {
// return true;
// }
// }
// return true;
// }
//
// /**
// * Visitor that gets a collection of top level objects to be scheduled, such
// * as root directories (if there is FS) or LayoutFiles and virtual
// * directories, also if there is no FS.
// */
// static class GetRootDirectoryVisitor extends GetFilesContentVisitor {
//
// @Override
// public Collection<AbstractFile> visit(VirtualDirectory ld) {
// //case when we hit a layout directoryor local file container, not under a real FS
// //or when root virt dir is scheduled
// Collection<AbstractFile> ret = new ArrayList<>();
// ret.add(ld);
// return ret;
// }
//
// @Override
// public Collection<AbstractFile> visit(LayoutFile lf) {
// //case when we hit a layout file, not under a real FS
// Collection<AbstractFile> ret = new ArrayList<>();
// ret.add(lf);
// return ret;
// }
//
// @Override
// public Collection<AbstractFile> visit(Directory drctr) {
// //we hit a real directory, a child of real FS
// Collection<AbstractFile> ret = new ArrayList<>();
// ret.add(drctr);
// return ret;
// }
//
// @Override
// public Collection<AbstractFile> visit(FileSystem fs) {
// return getAllFromChildren(fs);
// }
//
// @Override
// public Collection<AbstractFile> visit(File file) {
// //can have derived files
// return getAllFromChildren(file);
// }
//
// @Override
// public Collection<AbstractFile> visit(DerivedFile derivedFile) {
// //can have derived files
// //TODO test this and overall scheduler with derived files
// return getAllFromChildren(derivedFile);
// }
//
// @Override
// public Collection<AbstractFile> visit(LocalFile localFile) {
// //can have local files
// //TODO test this and overall scheduler with local files
// return getAllFromChildren(localFile);
// }
// }
//
// /**
// * Root directory sorter
// */
// private static class RootDirectoryTaskComparator implements Comparator<FileIngestTask> {
//
// @Override
// public int compare(FileIngestTask q1, FileIngestTask q2) {
// AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile());
// AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile());
// if (p1 == p2) {
// return (int) (q2.getFile().getId() - q1.getFile().getId());
// } else {
// return p2.ordinal() - p1.ordinal();
// }
// }
//
// /**
// * Priority determination for sorted AbstractFile, used by
// * RootDirComparator
// */
// private static class AbstractFilePriority {
//
// enum Priority {
//
// LAST, LOW, MEDIUM, HIGH
// }
// static final List<Pattern> LAST_PRI_PATHS = new ArrayList<>();
// static final List<Pattern> LOW_PRI_PATHS = new ArrayList<>();
// static final List<Pattern> MEDIUM_PRI_PATHS = new ArrayList<>();
// static final List<Pattern> HIGH_PRI_PATHS = new ArrayList<>();
// /* prioritize root directory folders based on the assumption that we are
// * looking for user content. Other types of investigations may want different
// * priorities. */
//
// static /* prioritize root directory folders based on the assumption that we are
// * looking for user content. Other types of investigations may want different
// * priorities. */ {
// // these files have no structure, so they go last
// //unalloc files are handled as virtual files in getPriority()
// //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE));
// //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE));
// LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE));
// LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE));
// // orphan files are often corrupt and windows does not typically have
// // user content, so put them towards the bottom
// LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE));
// LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE));
// // all other files go into the medium category too
// MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE));
// // user content is top priority
// HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE));
// HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE));
// HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE));
// HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE));
// }
//
// /**
// * Get the scheduling priority for a given file.
// *
// * @param abstractFile
// * @return
// */
// static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) {
// if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
// //quickly filter out unstructured content
// //non-fs virtual files and dirs, such as representing unalloc space
// return AbstractFilePriority.Priority.LAST;
// }
// //determine the fs files priority by name
// final String path = abstractFile.getName();
// if (path == null) {
// return AbstractFilePriority.Priority.MEDIUM;
// }
// for (Pattern p : HIGH_PRI_PATHS) {
// Matcher m = p.matcher(path);
// if (m.find()) {
// return AbstractFilePriority.Priority.HIGH;
// }
// }
// for (Pattern p : MEDIUM_PRI_PATHS) {
// Matcher m = p.matcher(path);
// if (m.find()) {
// return AbstractFilePriority.Priority.MEDIUM;
// }
// }
// for (Pattern p : LOW_PRI_PATHS) {
// Matcher m = p.matcher(path);
// if (m.find()) {
// return AbstractFilePriority.Priority.LOW;
// }
// }
// for (Pattern p : LAST_PRI_PATHS) {
// Matcher m = p.matcher(path);
// if (m.find()) {
// return AbstractFilePriority.Priority.LAST;
// }
// }
// //default is medium
// return AbstractFilePriority.Priority.MEDIUM;
// }
// }
// }
//}

88
Core/src/org/sleuthkit/autopsy/ingest/GetRootDirectoryVisitor.java Executable file
View File

@ -0,0 +1,88 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Collection;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.VirtualDirectory;
/**
* Finds top level objects such as file system root directories, layout
* files and virtual directories.
*/
final class GetRootDirectoryVisitor extends GetFilesContentVisitor {
@Override
public Collection<AbstractFile> visit(VirtualDirectory ld) {
//case when we hit a layout directoryor local file container, not under a real FS
//or when root virt dir is scheduled
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(ld);
return ret;
}
@Override
public Collection<AbstractFile> visit(LayoutFile lf) {
//case when we hit a layout file, not under a real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(lf);
return ret;
}
@Override
public Collection<AbstractFile> visit(Directory drctr) {
//we hit a real directory, a child of real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(drctr);
return ret;
}
@Override
public Collection<AbstractFile> visit(FileSystem fs) {
return getAllFromChildren(fs);
}
@Override
public Collection<AbstractFile> visit(File file) {
//can have derived files
return getAllFromChildren(file);
}
@Override
public Collection<AbstractFile> visit(DerivedFile derivedFile) {
//can have derived files
//TODO test this and overall scheduler with derived files
return getAllFromChildren(derivedFile);
}
@Override
public Collection<AbstractFile> visit(LocalFile localFile) {
//can have local files
//TODO test this and overall scheduler with local files
return getAllFromChildren(localFile);
}
}

80
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
View File

@ -20,10 +20,7 @@ package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level; import java.util.logging.Level;
import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory; import org.netbeans.api.progress.ProgressHandleFactory;
@ -33,33 +30,26 @@ import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
/**
* Encapsulates a data source and the ingest module pipelines to be used to
* ingest the data source.
*/
final class IngestJob { final class IngestJob {
private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private static final ConcurrentHashMap<Long, IngestJob> ingestJobsById = new ConcurrentHashMap<>();
private final long id; private final long id;
private final Content rootDataSource; private final Content rootDataSource;
private final List<IngestModuleTemplate> ingestModuleTemplates; private final List<IngestModuleTemplate> ingestModuleTemplates;
private final boolean processUnallocatedSpace; private final boolean processUnallocatedSpace;
private final LinkedBlockingQueue<DataSourceIngestPipeline> dataSourceIngestPipelines = new LinkedBlockingQueue<>(); private final LinkedBlockingQueue<DataSourceIngestPipeline> dataSourceIngestPipelines = new LinkedBlockingQueue<>();
private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelines = new LinkedBlockingQueue<>(); private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelines = new LinkedBlockingQueue<>();
private final AtomicInteger tasksInProgress = new AtomicInteger(0); private long estimatedFilesToProcess = 0L; // Guarded by this
private final AtomicLong processedFiles = new AtomicLong(0L); private long processedFiles = 0L; // Guarded by this
private final AtomicLong filesToIngestEstimate = new AtomicLong(0L);
private ProgressHandle dataSourceTasksProgress; private ProgressHandle dataSourceTasksProgress;
private ProgressHandle fileTasksProgress; private ProgressHandle fileTasksProgress;
private volatile boolean cancelled; private volatile boolean cancelled = false;
IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) { IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) {
this.id = id; this.id = id;
this.rootDataSource = dataSource; this.rootDataSource = dataSource;
this.ingestModuleTemplates = ingestModuleTemplates; this.ingestModuleTemplates = ingestModuleTemplates;
this.processUnallocatedSpace = processUnallocatedSpace; this.processUnallocatedSpace = processUnallocatedSpace;
this.cancelled = false;
} }
long getId() { long getId() {
@ -73,8 +63,8 @@ final class IngestJob {
List<IngestModuleError> startUp() throws InterruptedException { List<IngestModuleError> startUp() throws InterruptedException {
List<IngestModuleError> errors = startUpIngestPipelines(); List<IngestModuleError> errors = startUpIngestPipelines();
if (errors.isEmpty()) { if (errors.isEmpty()) {
startDataSourceIngestProgressBar();
startFileIngestProgressBar(); startFileIngestProgressBar();
startDataSourceIngestProgressBar();
} }
return errors; return errors;
} }
@ -105,7 +95,8 @@ final class IngestJob {
} }
} }
return errors; logIngestModuleErrors(errors);
return errors; // Returned so UI can report to user.
} }
private void startDataSourceIngestProgressBar() { private void startDataSourceIngestProgressBar() {
@ -145,18 +136,9 @@ final class IngestJob {
return true; return true;
} }
}); });
long initialFilesCount = rootDataSource.accept(new GetFilesCountVisitor()); estimatedFilesToProcess = rootDataSource.accept(new GetFilesCountVisitor());
filesToIngestEstimate.getAndAdd(initialFilesCount);
fileTasksProgress.start(); fileTasksProgress.start();
fileTasksProgress.switchToDeterminate((int) initialFilesCount); // RJCTODO: This cast is troublesome, can use intValue fileTasksProgress.switchToDeterminate((int) estimatedFilesToProcess);
}
/**
* Called by the ingest task schedulers when an ingest task is added to this
* ingest job.
*/
void notifyTaskAdded() {
tasksInProgress.incrementAndGet();
} }
void process(Content dataSource) throws InterruptedException { void process(Content dataSource) throws InterruptedException {
@ -172,7 +154,6 @@ final class IngestJob {
} }
dataSourceIngestPipelines.put(pipeline); dataSourceIngestPipelines.put(pipeline);
} }
shutDownIfAllTasksCompleted();
} }
void process(AbstractFile file) throws InterruptedException { void process(AbstractFile file) throws InterruptedException {
@ -181,35 +162,37 @@ final class IngestJob {
// shut down check needs to occur. // shut down check needs to occur.
if (!isCancelled()) { if (!isCancelled()) {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
synchronized (this) {
++processedFiles;
if (processedFiles <= estimatedFilesToProcess) {
fileTasksProgress.progress(file.getName(), (int) processedFiles);
} else {
fileTasksProgress.progress(file.getName(), (int) estimatedFilesToProcess);
}
}
FileIngestPipeline pipeline = fileIngestPipelines.take(); FileIngestPipeline pipeline = fileIngestPipelines.take();
fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet());
errors.addAll(pipeline.process(file)); errors.addAll(pipeline.process(file));
fileIngestPipelines.put(pipeline); fileIngestPipelines.put(pipeline);
if (!errors.isEmpty()) { if (!errors.isEmpty()) {
logIngestModuleErrors(errors); logIngestModuleErrors(errors);
} }
} }
shutDownIfAllTasksCompleted();
} }
private void shutDownIfAllTasksCompleted() { void shutDown() {
if (tasksInProgress.decrementAndGet() == 0) { List<IngestModuleError> errors = new ArrayList<>();
List<IngestModuleError> errors = new ArrayList<>(); while (!dataSourceIngestPipelines.isEmpty()) {
while (!dataSourceIngestPipelines.isEmpty()) { DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll();
DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll(); errors.addAll(pipeline.shutDown());
errors.addAll(pipeline.shutDown()); }
} while (!fileIngestPipelines.isEmpty()) {
while (!fileIngestPipelines.isEmpty()) { FileIngestPipeline pipeline = fileIngestPipelines.poll();
FileIngestPipeline pipeline = fileIngestPipelines.poll(); errors.addAll(pipeline.shutDown());
errors.addAll(pipeline.shutDown()); }
} fileTasksProgress.finish();
fileTasksProgress.finish(); dataSourceTasksProgress.finish();
dataSourceTasksProgress.finish(); if (!errors.isEmpty()) {
ingestJobsById.remove(id); logIngestModuleErrors(errors);
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
IngestManager.getInstance().fireIngestJobCompleted(id);
} }
} }
@ -225,7 +208,8 @@ final class IngestJob {
void cancel() { void cancel() {
cancelled = true; cancelled = true;
fileTasksProgress.finish(); // RJCTODO: What about the other progress bar? fileTasksProgress.finish();
dataSourceTasksProgress.finish();
IngestManager.getInstance().fireIngestJobCancelled(id); IngestManager.getInstance().fireIngestJobCancelled(id);
} }
} }

2
Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java
View File

@ -60,7 +60,7 @@ public final class IngestJobContext {
*/ */
public void addFiles(List<AbstractFile> files) { public void addFiles(List<AbstractFile> files) {
for (AbstractFile file : files) { for (AbstractFile file : files) {
IngestJobScheduler.getInstance().addFileToIngestJob(ingestJob, file); IngestScheduler.getInstance().addFileToIngestJob(ingestJob, file);
} }
} }
} }

12
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -48,23 +48,23 @@ public class IngestManager {
private static final int MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS = 1; private static final int MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS = 1;
private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS
private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1;
private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 4; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16;
private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2;
private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class);
private static final IngestManager instance = new IngestManager(); private static final IngestManager instance = new IngestManager();
private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class);
private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class);
private final IngestJobScheduler scheduler = IngestJobScheduler.getInstance(); private final IngestScheduler scheduler = IngestScheduler.getInstance();
private final IngestMonitor ingestMonitor = new IngestMonitor(); private final IngestMonitor ingestMonitor = new IngestMonitor();
private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor();
private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor();
private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS);
private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor();
private final AtomicLong nextThreadId = new AtomicLong(0L);
private final ConcurrentHashMap<Long, Future<Void>> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ConcurrentHashMap<Long, Future<Void>> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final ConcurrentHashMap<Long, Future<?>> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ConcurrentHashMap<Long, Future<?>> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final ConcurrentHashMap<Long, Future<?>> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ConcurrentHashMap<Long, Future<?>> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final AtomicLong nextThreadId = new AtomicLong(0L);
private volatile IngestMessageTopComponent ingestMessageBox; private volatile IngestMessageTopComponent ingestMessageBox;
/** /**
@ -186,8 +186,8 @@ public class IngestManager {
} }
long taskId = nextThreadId.incrementAndGet(); long taskId = nextThreadId.incrementAndGet();
Future<?> task = startIngestJobsThreadPool.submit(new StartIngestJobsThread(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); Future<Void> task = startIngestJobsThreadPool.submit(new StartIngestJobsThread(taskId, dataSources, moduleTemplates, processUnallocatedSpace));
fileIngestThreads.put(taskId, task); startIngestJobThreads.put(taskId, task);
if (ingestMessageBox != null) { if (ingestMessageBox != null) {
ingestMessageBox.restoreMessages(); ingestMessageBox.restoreMessages();
@ -218,7 +218,6 @@ public class IngestManager {
logger.log(Level.SEVERE, "Unexpected thread interrupt", ex); logger.log(Level.SEVERE, "Unexpected thread interrupt", ex);
} }
} }
startIngestJobThreads.clear(); // Make sure.
// Cancel all the jobs already created. This will make the the ingest // Cancel all the jobs already created. This will make the the ingest
// threads flush out any lingering ingest tasks without processing them. // threads flush out any lingering ingest tasks without processing them.
@ -511,6 +510,7 @@ public class IngestManager {
try { try {
IngestTask task = tasks.getNextTask(); // Blocks. IngestTask task = tasks.getNextTask(); // Blocks.
task.execute(); task.execute();
scheduler.ingestTaskIsCompleted(task);
} catch (InterruptedException ex) { } catch (InterruptedException ex) {
break; break;
} }

160
Core/src/org/sleuthkit/autopsy/ingest/IngestJobScheduler.java → Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java
View File

@ -32,35 +32,31 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File; import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem; import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.VirtualDirectory;
final class IngestJobScheduler { final class IngestScheduler {
private static final Logger logger = Logger.getLogger(IngestJobScheduler.class.getName()); private static final IngestScheduler instance = new IngestScheduler();
private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName());
private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
private static IngestJobScheduler instance = new IngestJobScheduler();
private final AtomicLong nextIngestJobId = new AtomicLong(0L);
private final ConcurrentHashMap<Long, IngestJob> ingestJobsById = new ConcurrentHashMap<>(); private final ConcurrentHashMap<Long, IngestJob> ingestJobsById = new ConcurrentHashMap<>();
private final LinkedBlockingQueue<DataSourceIngestTask> dataSourceTasks = new LinkedBlockingQueue<>(); private final LinkedBlockingQueue<DataSourceIngestTask> dataSourceTasks = new LinkedBlockingQueue<>();
private final TreeSet<FileIngestTask> rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); private final TreeSet<FileIngestTask> rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); // Guarded by this
private final List<FileIngestTask> directoryTasks = new ArrayList<>(); private final List<FileIngestTask> directoryTasks = new ArrayList<>(); // Guarded by this
private final LinkedBlockingQueue<FileIngestTask> fileTasks = new LinkedBlockingQueue<>(); private final LinkedBlockingQueue<FileIngestTask> fileTasks = new LinkedBlockingQueue<>(); // Guarded by this
private final DataSourceIngestTaskQueue dataSourceIngestTaskQueue = new DataSourceIngestTaskQueue(); private final List<IngestTask> tasksInProgress = new ArrayList<>(); // Guarded by this
private final FileIngestTaskQueue fileIngestTaskQueue = new FileIngestTaskQueue(); private final DataSourceIngestTaskQueue dataSourceTaskDispenser = new DataSourceIngestTaskQueue();
private final FileIngestTaskQueue fileTaskDispenser = new FileIngestTaskQueue();
private final AtomicLong nextIngestJobId = new AtomicLong(0L);
static IngestJobScheduler getInstance() { static IngestScheduler getInstance() {
return instance; return instance;
} }
private IngestJobScheduler() { private IngestScheduler() {
} }
/** /**
@ -74,7 +70,7 @@ final class IngestJobScheduler {
* @return A collection of ingest module start up errors, empty on success. * @return A collection of ingest module start up errors, empty on success.
* @throws InterruptedException * @throws InterruptedException
*/ */
synchronized List<IngestModuleError> startIngestJob(Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException { List<IngestModuleError> startIngestJob(Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException {
long jobId = nextIngestJobId.incrementAndGet(); long jobId = nextIngestJobId.incrementAndGet();
IngestJob job = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace); IngestJob job = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace);
ingestJobsById.put(jobId, job); ingestJobsById.put(jobId, job);
@ -99,10 +95,11 @@ final class IngestJobScheduler {
} }
synchronized void addDataSourceToIngestJob(IngestJob job, Content dataSource) throws InterruptedException { synchronized void addDataSourceToIngestJob(IngestJob job, Content dataSource) throws InterruptedException {
// Enqueue a data source ingest task for the data source.
// If the thread executing this code is interrupted, it is because the // If the thread executing this code is interrupted, it is because the
// the number of ingest threads has been decreased while ingest jobs are // the number of ingest threads has been decreased while ingest jobs are
// running. The calling thread will exit in an orderly fashion, but the // running. The calling thread will exit in an orderly fashion, but the
// task still needs to be enqueued rather than lost. // task still needs to be enqueued rather than lost, hence the loop.
DataSourceIngestTask task = new DataSourceIngestTask(job, dataSource); DataSourceIngestTask task = new DataSourceIngestTask(job, dataSource);
while (true) { while (true) {
try { try {
@ -115,24 +112,27 @@ final class IngestJobScheduler {
} }
} }
// Get the top level files of the data source.
Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
List<AbstractFile> firstLevelFiles = new ArrayList<>(); List<AbstractFile> toptLevelFiles = new ArrayList<>();
if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
// The data source is file. // The data source is itself a file.
firstLevelFiles.add((AbstractFile) dataSource); toptLevelFiles.add((AbstractFile) dataSource);
} else { } else {
for (AbstractFile root : rootObjects) { for (AbstractFile root : rootObjects) {
List<Content> children; List<Content> children;
try { try {
children = root.getChildren(); children = root.getChildren();
if (children.isEmpty()) { if (children.isEmpty()) {
//add the root itself, could be unalloc file, child of volume or image // Add the root object itself, it could be an unallocated space
firstLevelFiles.add(root); // file, or a child of a volume or an image.
toptLevelFiles.add(root);
} else { } else {
//root for fs root dir, schedule children dirs/files // The root object is a file system root directory, get
// the files within it.
for (Content child : children) { for (Content child : children) {
if (child instanceof AbstractFile) { if (child instanceof AbstractFile) {
firstLevelFiles.add((AbstractFile) child); toptLevelFiles.add((AbstractFile) child);
} }
} }
} }
@ -141,26 +141,30 @@ final class IngestJobScheduler {
} }
} }
} }
for (AbstractFile firstLevelFile : firstLevelFiles) {
// Enqueue file ingest tasks for the top level files.
for (AbstractFile firstLevelFile : toptLevelFiles) {
FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile);
if (shouldEnqueueFileTask(fileTask)) { if (shouldEnqueueFileTask(fileTask)) {
rootDirectoryTasks.add(fileTask); rootDirectoryTasks.add(fileTask);
fileTask.getIngestJob().notifyTaskAdded();
} }
} }
// Reshuffle/update the dir and file level queues if needed updateFileTaskQueues(null);
updateFileTaskQueues();
} }
synchronized void addFileToIngestJob(IngestJob job, AbstractFile file) { // RJCTODO: Just one at a time? void addFileToIngestJob(IngestJob job, AbstractFile file) {
FileIngestTask task = new FileIngestTask(job, file); FileIngestTask task = new FileIngestTask(job, file);
if (shouldEnqueueFileTask(task)) { if (shouldEnqueueFileTask(task)) {
addTaskToFileQueue(task); addTaskToFileQueue(task);
} }
} }
private synchronized void updateFileTaskQueues() throws InterruptedException { private synchronized void updateFileTaskQueues(FileIngestTask taskInProgress) throws InterruptedException {
if (taskInProgress != null) {
tasksInProgress.add(taskInProgress);
}
// we loop because we could have a directory that has all files // we loop because we could have a directory that has all files
// that do not get enqueued // that do not get enqueued
while (true) { while (true) {
@ -194,7 +198,6 @@ final class IngestJobScheduler {
FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile); FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile);
if (childFile.hasChildren()) { if (childFile.hasChildren()) {
directoryTasks.add(childTask); directoryTasks.add(childTask);
childTask.getIngestJob().notifyTaskAdded();
} else if (shouldEnqueueFileTask(childTask)) { } else if (shouldEnqueueFileTask(childTask)) {
addTaskToFileQueue(childTask); addTaskToFileQueue(childTask);
} }
@ -223,13 +226,6 @@ final class IngestJobScheduler {
} }
} }
/**
* Check if the file is a special file that we should skip
*
* @param processTask a task whose file to check if should be queued of
* skipped
* @return true if should be enqueued, false otherwise
*/
private static boolean shouldEnqueueFileTask(final FileIngestTask processTask) { private static boolean shouldEnqueueFileTask(final FileIngestTask processTask) {
final AbstractFile aFile = processTask.getFile(); final AbstractFile aFile = processTask.getFile();
//if it's unalloc file, skip if so scheduled //if it's unalloc file, skip if so scheduled
@ -282,68 +278,52 @@ final class IngestJobScheduler {
} }
IngestTaskQueue getDataSourceIngestTaskQueue() { IngestTaskQueue getDataSourceIngestTaskQueue() {
return dataSourceIngestTaskQueue; return dataSourceTaskDispenser;
} }
IngestTaskQueue getFileIngestTaskQueue() { IngestTaskQueue getFileIngestTaskQueue() {
return fileIngestTaskQueue; return fileTaskDispenser;
} }
/** void ingestTaskIsCompleted(IngestTask completedTask) {
* Finds top level objects such as file system root directories, layout if (ingestJobIsCompleted(completedTask)) {
* files and virtual directories. IngestJob job = completedTask.getIngestJob();
*/ job.shutDown();
private static class GetRootDirectoryVisitor extends GetFilesContentVisitor { ingestJobsById.remove(job.getId());
IngestManager.getInstance().fireIngestJobCompleted(job.getId());
@Override
public Collection<AbstractFile> visit(VirtualDirectory ld) {
//case when we hit a layout directoryor local file container, not under a real FS
//or when root virt dir is scheduled
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(ld);
return ret;
} }
}
@Override private synchronized boolean ingestJobIsCompleted(IngestTask completedTask) {
public Collection<AbstractFile> visit(LayoutFile lf) { tasksInProgress.remove(completedTask);
//case when we hit a layout file, not under a real FS IngestJob job = completedTask.getIngestJob();
Collection<AbstractFile> ret = new ArrayList<>(); long jobId = job.getId();
ret.add(lf); for (IngestTask task : tasksInProgress) {
return ret; if (task.getIngestJob().getId() == jobId) {
return false;
}
} }
for (FileIngestTask task : fileTasks) {
@Override if (task.getIngestJob().getId() == jobId) {
public Collection<AbstractFile> visit(Directory drctr) { return false;
//we hit a real directory, a child of real FS }
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(drctr);
return ret;
} }
for (FileIngestTask task : directoryTasks) {
@Override if (task.getIngestJob().getId() == jobId) {
public Collection<AbstractFile> visit(FileSystem fs) { return false;
return getAllFromChildren(fs); }
} }
for (FileIngestTask task : rootDirectoryTasks) {
@Override if (task.getIngestJob().getId() == jobId) {
public Collection<AbstractFile> visit(File file) { return false;
//can have derived files }
return getAllFromChildren(file);
} }
for (DataSourceIngestTask task : dataSourceTasks) {
@Override if (task.getIngestJob().getId() == jobId) {
public Collection<AbstractFile> visit(DerivedFile derivedFile) { return false;
//can have derived files }
//TODO test this and overall scheduler with derived files
return getAllFromChildren(derivedFile);
}
@Override
public Collection<AbstractFile> visit(LocalFile localFile) {
//can have local files
//TODO test this and overall scheduler with local files
return getAllFromChildren(localFile);
} }
return true;
} }
private static class RootDirectoryTaskComparator implements Comparator<FileIngestTask> { private static class RootDirectoryTaskComparator implements Comparator<FileIngestTask> {
@ -455,7 +435,7 @@ final class IngestJobScheduler {
@Override @Override
public IngestTask getNextTask() throws InterruptedException { public IngestTask getNextTask() throws InterruptedException {
FileIngestTask task = fileTasks.take(); FileIngestTask task = fileTasks.take();
updateFileTaskQueues(); updateFileTaskQueues(task);
return task; return task;
} }
} }

15
Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java
View File

@ -18,6 +18,17 @@
*/ */
package org.sleuthkit.autopsy.ingest; package org.sleuthkit.autopsy.ingest;
interface IngestTask { abstract class IngestTask {
void execute() throws InterruptedException;
private final IngestJob job;
IngestTask(IngestJob job) {
this.job = job;
}
IngestJob getIngestJob() {
return job;
}
abstract void execute() throws InterruptedException;
} }