diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 97f082cce3..63152f4199 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -719,7 +719,7 @@ final class DataSourceIngestJob { * @param files A list of the files to add. */ void addFiles(List files) { - if (DataSourceIngestJob.Stages.FIRST == this.stage) { // RJCTODO: Is this synchronized correctly + if (DataSourceIngestJob.Stages.FIRST == this.stage) { for (AbstractFile file : files) { DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file); } @@ -970,7 +970,7 @@ final class DataSourceIngestJob { /** * Stores basic diagnostic statistics for a data source ingest job. */ - class Snapshot { + final class Snapshot { private final String dataSource; private final long jobId; @@ -1129,7 +1129,7 @@ final class DataSourceIngestJob { return this.tasksSnapshot.getRunningListSize(); } - boolean getCancelled() { + boolean isCancelled() { return this.jobCancelled; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 8218186b4c..961428148e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -20,10 +20,12 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.sleuthkit.datamodel.Content; @@ -38,7 +40,8 @@ public final class IngestJob { private static final AtomicLong nextId = new AtomicLong(0L); private final long id; private final Map dataSourceJobs; - private boolean cancelled; + private final AtomicInteger incompleteJobsCount; + private volatile boolean cancelled; /** * Constructs an ingest job that runs a collection of data sources through a @@ -51,11 +54,12 @@ public final class IngestJob { */ IngestJob(Collection dataSources, IngestJobSettings settings, boolean runInteractively) { this.id = IngestJob.nextId.getAndIncrement(); - this.dataSourceJobs = new HashMap<>(); + this.dataSourceJobs = new ConcurrentHashMap<>(); for (Content dataSource : dataSources) { DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings, runInteractively); this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); } + incompleteJobsCount = new AtomicInteger(dataSourceJobs.size()); } /** @@ -73,7 +77,7 @@ public final class IngestJob { * * @return True or false. */ - synchronized boolean hasIngestPipeline() { + boolean hasIngestPipeline() { for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { if (dataSourceJob.hasIngestPipeline()) { return true; @@ -88,16 +92,29 @@ public final class IngestJob { * * @return A collection of ingest module start up errors, empty on success. */ - synchronized List start() { + List start() { List errors = new ArrayList<>(); for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { errors.addAll(dataSourceJob.start()); if (!errors.isEmpty()) { - // RJCTODO: Need to let sucessfully started data source ingest - // jobs know they should shut down. break; } } + + /** + * TODO: Need to let successfully started data source ingest jobs know + * they should shut down. This means that the start up of the ingest + * module pipelines and the submission of ingest tasks should be + * separated. This cancellation is just a stop gap; fortunately, if + * startup is going to fail, it will likely fail for the first child + * data source ingest job. + */ + if (!errors.isEmpty()) { + for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + dataSourceJob.cancel(); + } + } + return errors; } @@ -106,26 +123,8 @@ public final class IngestJob { * * @return The snapshot. */ - synchronized public ProgressSnapshot getSnapshot() { - DataSourceIngestModuleHandle moduleHandle = null; - boolean fileIngestIsRunning = false; - Date fileIngestStartTime = null; - for (DataSourceIngestJob.Snapshot snapshot : this.getDataSourceIngestJobSnapshots()) { - if (null == moduleHandle) { - DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); - if (null != module) { - moduleHandle = new DataSourceIngestModuleHandle(this.dataSourceJobs.get(snapshot.getJobId()), module); - } - } - if (snapshot.fileIngestIsRunning()) { - fileIngestIsRunning = true; - } - Date childFileIngestStartTime = snapshot.fileIngestStartTime(); - if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) { - fileIngestStartTime = childFileIngestStartTime; - } - } - return new ProgressSnapshot(moduleHandle, fileIngestIsRunning, fileIngestStartTime, this.cancelled); + public ProgressSnapshot getSnapshot() { + return new ProgressSnapshot(); } /** @@ -134,7 +133,7 @@ public final class IngestJob { * * @return A list of data source ingest job progress snapshots. */ - synchronized List getDataSourceIngestJobSnapshots() { + List getDataSourceIngestJobSnapshots() { List snapshots = new ArrayList<>(); for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { snapshots.add(dataSourceJob.getSnapshot()); @@ -148,7 +147,7 @@ public final class IngestJob { * but there may be a delay before all of the ingest modules in the * pipelines respond by stopping processing. */ - synchronized public void cancel() { + public void cancel() { for (DataSourceIngestJob job : this.dataSourceJobs.values()) { job.cancel(); } @@ -161,7 +160,7 @@ public final class IngestJob { * * @return True or false. */ - synchronized public boolean isCancelled() { + public boolean isCancelled() { return this.cancelled; } @@ -171,9 +170,8 @@ public final class IngestJob { * * @param dataSourceIngestJob A completed data source ingest job. */ - synchronized void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { - this.dataSourceJobs.remove(dataSourceIngestJob.getId()); - if (this.dataSourceJobs.isEmpty()) { + void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) { + if (incompleteJobsCount.decrementAndGet() == 0) { IngestManager.getInstance().finishIngestJob(this); } } @@ -181,30 +179,85 @@ public final class IngestJob { /** * A snapshot of the progress of an ingest job. */ - public static final class ProgressSnapshot { + public final class ProgressSnapshot { - private final DataSourceIngestModuleHandle dataSourceModule; - private final boolean fileIngestRunning; - private final Date fileIngestStartTime; - private final boolean cancelled; + private final List dataSourceProcessingSnapshots; + private DataSourceIngestModuleHandle dataSourceModule; + private boolean fileIngestRunning; + private Date fileIngestStartTime; + private final boolean jobCancelled; + + /** + * A snapshot of the progress of an ingest job on the processing of a + * data source. + */ + public final class DataSourceProcessingSnapshot { + + private final DataSourceIngestJob.Snapshot snapshot; + + private DataSourceProcessingSnapshot(DataSourceIngestJob.Snapshot snapshot) { + this.snapshot = snapshot; + } + + /** + * Gets the name of the data source that is the subject of this + * snapshot. + * + * @return A data source name string. + */ + public String getDataSource() { + return snapshot.getDataSource(); + } + + /** + * Indicates whether or not the processing of the data source that + * is the subject of this snapshot was canceled. + * + * @return True or false. + */ + public boolean isCancelled() { + return snapshot.isCancelled(); + } + + /** + * Gets a list of the display names of any canceled data source + * level ingest modules. + * + * @return A list of canceled data source level ingest module + * display names, possibly empty. + */ + public List getCancelledDataSourceIngestModules() { + return snapshot.getCancelledDataSourceIngestModules(); + } + + } /** * Constructs a snapshot of ingest job progress. - * - * @param dataSourceModule The currently running data source level - * ingest module, may be null. - * @param fileIngestRunning Whether or not file ingest is currently - * running. - * @param fileIngestStartTime The start time of file level ingest, may - * be null. - * @param cancelled Whether or not a cancellation request has been - * issued. */ - private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean cancelled) { - this.dataSourceModule = dataSourceModule; - this.fileIngestRunning = fileIngestRunning; - this.fileIngestStartTime = fileIngestStartTime; - this.cancelled = cancelled; + private ProgressSnapshot() { + dataSourceModule = null; + fileIngestRunning = false; + fileIngestStartTime = null; + dataSourceProcessingSnapshots = new ArrayList<>(); + for (DataSourceIngestJob dataSourceJob : dataSourceJobs.values()) { + DataSourceIngestJob.Snapshot snapshot = dataSourceJob.getSnapshot(); + dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot)); + if (null == dataSourceModule) { + DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); + if (null != module) { + dataSourceModule = new DataSourceIngestModuleHandle(dataSourceJobs.get(snapshot.getJobId()), module); + } + } + if (snapshot.fileIngestIsRunning()) { + fileIngestRunning = true; + } + Date childFileIngestStartTime = snapshot.fileIngestStartTime(); + if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) { + fileIngestStartTime = childFileIngestStartTime; + } + } + this.jobCancelled = cancelled; } /** @@ -243,7 +296,16 @@ public final class IngestJob { * @return True or false. */ public boolean isCancelled() { - return this.cancelled; + return this.jobCancelled; + } + + /** + * Gets snapshots of the progress processing individual data sources. + * + * @return The list of snapshots. + */ + public List getDataSourceSnapshots() { + return Collections.unmodifiableList(this.dataSourceProcessingSnapshots); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index b0cb987710..a1cb07858e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -404,7 +404,7 @@ public class IngestManager { boolean success = false; if (this.jobCreationIsEnabled) { - if (runInteractively && jobsById.isEmpty()) { // RJCTODO: This is sort of broken + if (runInteractively && jobsById.isEmpty()) { clearIngestMessageBox(); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 55e19676aa..96784a86c7 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -149,7 +149,7 @@ class QueryResults { if (writeResult != null) { newArtifacts.add(writeResult.getArtifact()); if (notifyInbox) { - writeSingleFileInboxMessage(writeResult, hit.getContent()); // RJCTODO: Consider rewriting this message post code + writeSingleFileInboxMessage(writeResult, hit.getContent()); } } else { logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hit.getContent(), keyword.toString()}); //NON-NLS diff --git a/ScalpelCarver/nbproject/platform.properties b/ScalpelCarver/nbproject/platform.properties new file mode 100755 index 0000000000..1554e42acf --- /dev/null +++ b/ScalpelCarver/nbproject/platform.properties @@ -0,0 +1,123 @@ +branding.token=autopsy +# Version of platform that is automatically downloaded +# Note build.xml has similar definitions that should be kept in sync (manually) +netbeans-plat-version=7.3.1 +suite.dir=${basedir} +nbplatform.active.dir=${suite.dir}/netbeans-plat/${netbeans-plat-version} +harness.dir=${nbplatform.active.dir}/harness +bootstrap.url=http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/netbeans/harness/tasks.jar +# Where we get the platform from. To see what versions are available, open URL in browser up to the .../updates part of the URL +autoupdate.catalog.url=http://dlc.sun.com.edgesuite.net/netbeans/updates/${netbeans-plat-version}/uc/final/distribution/catalog.xml.gz +cluster.path=\ + ${nbplatform.active.dir}/harness:\ + ${nbplatform.active.dir}/java:\ + ${nbplatform.active.dir}/platform +disabled.modules=\ + org.apache.tools.ant.module,\ + org.netbeans.api.debugger.jpda,\ + org.netbeans.api.java,\ + org.netbeans.lib.nbjavac,\ + org.netbeans.libs.cglib,\ + org.netbeans.libs.javacapi,\ + org.netbeans.libs.javacimpl,\ + org.netbeans.libs.springframework,\ + org.netbeans.modules.ant.browsetask,\ + org.netbeans.modules.ant.debugger,\ + org.netbeans.modules.ant.freeform,\ + org.netbeans.modules.ant.grammar,\ + org.netbeans.modules.ant.kit,\ + org.netbeans.modules.beans,\ + org.netbeans.modules.classfile,\ + org.netbeans.modules.dbschema,\ + org.netbeans.modules.debugger.jpda,\ + org.netbeans.modules.debugger.jpda.ant,\ + org.netbeans.modules.debugger.jpda.kit,\ + org.netbeans.modules.debugger.jpda.projects,\ + org.netbeans.modules.debugger.jpda.ui,\ + org.netbeans.modules.debugger.jpda.visual,\ + org.netbeans.modules.findbugs.installer,\ + org.netbeans.modules.form,\ + org.netbeans.modules.form.binding,\ + org.netbeans.modules.form.j2ee,\ + org.netbeans.modules.form.kit,\ + org.netbeans.modules.form.nb,\ + org.netbeans.modules.form.refactoring,\ + org.netbeans.modules.hibernate,\ + org.netbeans.modules.hibernatelib,\ + org.netbeans.modules.hudson.ant,\ + org.netbeans.modules.hudson.maven,\ + org.netbeans.modules.i18n,\ + org.netbeans.modules.i18n.form,\ + org.netbeans.modules.j2ee.core.utilities,\ + org.netbeans.modules.j2ee.eclipselink,\ + org.netbeans.modules.j2ee.eclipselinkmodelgen,\ + org.netbeans.modules.j2ee.jpa.refactoring,\ + org.netbeans.modules.j2ee.jpa.verification,\ + org.netbeans.modules.j2ee.metadata,\ + org.netbeans.modules.j2ee.metadata.model.support,\ + org.netbeans.modules.j2ee.persistence,\ + org.netbeans.modules.j2ee.persistence.kit,\ + org.netbeans.modules.j2ee.persistenceapi,\ + org.netbeans.modules.java.api.common,\ + org.netbeans.modules.java.debug,\ + org.netbeans.modules.java.editor,\ + org.netbeans.modules.java.editor.lib,\ + org.netbeans.modules.java.examples,\ + org.netbeans.modules.java.freeform,\ + org.netbeans.modules.java.guards,\ + org.netbeans.modules.java.helpset,\ + org.netbeans.modules.java.hints,\ + org.netbeans.modules.java.hints.declarative,\ + org.netbeans.modules.java.hints.declarative.test,\ + org.netbeans.modules.java.hints.legacy.spi,\ + org.netbeans.modules.java.hints.test,\ + org.netbeans.modules.java.hints.ui,\ + org.netbeans.modules.java.j2seplatform,\ + org.netbeans.modules.java.j2seproject,\ + org.netbeans.modules.java.kit,\ + org.netbeans.modules.java.lexer,\ + org.netbeans.modules.java.navigation,\ + org.netbeans.modules.java.platform,\ + org.netbeans.modules.java.preprocessorbridge,\ + org.netbeans.modules.java.project,\ + org.netbeans.modules.java.source,\ + org.netbeans.modules.java.source.ant,\ + org.netbeans.modules.java.source.queries,\ + org.netbeans.modules.java.source.queriesimpl,\ + org.netbeans.modules.java.sourceui,\ + org.netbeans.modules.java.testrunner,\ + org.netbeans.modules.javadoc,\ + org.netbeans.modules.javawebstart,\ + org.netbeans.modules.junit,\ + org.netbeans.modules.maven,\ + org.netbeans.modules.maven.checkstyle,\ + org.netbeans.modules.maven.coverage,\ + org.netbeans.modules.maven.embedder,\ + org.netbeans.modules.maven.grammar,\ + org.netbeans.modules.maven.graph,\ + org.netbeans.modules.maven.hints,\ + org.netbeans.modules.maven.indexer,\ + org.netbeans.modules.maven.junit,\ + org.netbeans.modules.maven.kit,\ + org.netbeans.modules.maven.model,\ + org.netbeans.modules.maven.osgi,\ + org.netbeans.modules.maven.persistence,\ + org.netbeans.modules.maven.refactoring,\ + org.netbeans.modules.maven.repository,\ + org.netbeans.modules.maven.search,\ + org.netbeans.modules.maven.spring,\ + org.netbeans.modules.projectimport.eclipse.core,\ + org.netbeans.modules.projectimport.eclipse.j2se,\ + org.netbeans.modules.refactoring.java,\ + org.netbeans.modules.spellchecker.bindings.java,\ + org.netbeans.modules.spring.beans,\ + org.netbeans.modules.testng,\ + org.netbeans.modules.testng.ant,\ + org.netbeans.modules.testng.maven,\ + org.netbeans.modules.websvc.jaxws21,\ + org.netbeans.modules.websvc.jaxws21api,\ + org.netbeans.modules.websvc.saas.codegen.java,\ + org.netbeans.modules.xml.jaxb,\ + org.netbeans.modules.xml.tools.java,\ + org.netbeans.spi.java.hints + diff --git a/ScalpelCarver/nbproject/project.xml b/ScalpelCarver/nbproject/project.xml index a1951a32d9..866c390d45 100644 --- a/ScalpelCarver/nbproject/project.xml +++ b/ScalpelCarver/nbproject/project.xml @@ -4,7 +4,7 @@ org.sleuthkit.autopsy.scalpel - + org.openide.util diff --git a/ScalpelCarver/nbproject/suite.properties b/ScalpelCarver/nbproject/suite.properties deleted file mode 100644 index 29d7cc9bd6..0000000000 --- a/ScalpelCarver/nbproject/suite.properties +++ /dev/null @@ -1 +0,0 @@ -suite.dir=${basedir}/.. diff --git a/nbproject/project.properties b/nbproject/project.properties index b63b854039..128c4101c0 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -27,13 +27,11 @@ modules=\ ${project.org.sleuthkit.autopsy.testing}:\ ${project.org.sleuthkit.autopsy.thunderbirdparser}:\ ${project.org.sleuthkit.autopsy.core}:\ - ${project.org.sleuthkit.autopsy.corelibs}:\ - ${project.org.sleuthkit.autopsy.scalpel} + ${project.org.sleuthkit.autopsy.corelibs} project.org.sleuthkit.autopsy.core=Core project.org.sleuthkit.autopsy.corelibs=CoreLibs project.org.sleuthkit.autopsy.keywordsearch=KeywordSearch project.org.sleuthkit.autopsy.recentactivity=RecentActivity project.org.sleuthkit.autopsy.testing=Testing project.org.sleuthkit.autopsy.thunderbirdparser=thunderbirdparser -project.org.sleuthkit.autopsy.scalpel=ScalpelCarver