Merge remote-tracking branch 'upstream/develop' into stix

Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java

@@ -719,7 +719,7 @@ final class DataSourceIngestJob {
      * @param files A list of the files to add.
      */
     void addFiles(List<AbstractFile> files) {
-        if (DataSourceIngestJob.Stages.FIRST == this.stage) { // RJCTODO: Is this synchronized correctly
+        if (DataSourceIngestJob.Stages.FIRST == this.stage) {
             for (AbstractFile file : files) {
                 DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file);
             }
@@ -970,7 +970,7 @@ final class DataSourceIngestJob {
     /**
      * Stores basic diagnostic statistics for a data source ingest job.
      */
-    class Snapshot {
+    final class Snapshot {
 
         private final String dataSource;
         private final long jobId;
@@ -1129,7 +1129,7 @@ final class DataSourceIngestJob {
             return this.tasksSnapshot.getRunningListSize();
         }
 
-        boolean getCancelled() {
+        boolean isCancelled() {
             return this.jobCancelled;
         }
         

Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java

@@ -20,10 +20,12 @@ package org.sleuthkit.autopsy.ingest;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import org.sleuthkit.datamodel.Content;
 
@@ -38,7 +40,8 @@ public final class IngestJob {
     private static final AtomicLong nextId = new AtomicLong(0L);
     private final long id;
     private final Map<Long, DataSourceIngestJob> dataSourceJobs;
-    private boolean cancelled;
+    private final AtomicInteger incompleteJobsCount;
+    private volatile boolean cancelled;
 
     /**
      * Constructs an ingest job that runs a collection of data sources through a
@@ -51,11 +54,12 @@ public final class IngestJob {
      */
     IngestJob(Collection<Content> dataSources, IngestJobSettings settings, boolean runInteractively) {
         this.id = IngestJob.nextId.getAndIncrement();
-        this.dataSourceJobs = new HashMap<>();
+        this.dataSourceJobs = new ConcurrentHashMap<>();
         for (Content dataSource : dataSources) {
             DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings, runInteractively);
             this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob);
         }
+        incompleteJobsCount = new AtomicInteger(dataSourceJobs.size());
     }
 
     /**
@@ -73,7 +77,7 @@ public final class IngestJob {
      *
      * @return True or false.
      */
-    synchronized boolean hasIngestPipeline() {
+    boolean hasIngestPipeline() {
         for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) {
             if (dataSourceJob.hasIngestPipeline()) {
                 return true;
@@ -88,16 +92,29 @@ public final class IngestJob {
      *
      * @return A collection of ingest module start up errors, empty on success.
      */
-    synchronized List<IngestModuleError> start() {
+    List<IngestModuleError> start() {
         List<IngestModuleError> errors = new ArrayList<>();
         for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) {
             errors.addAll(dataSourceJob.start());
             if (!errors.isEmpty()) {
-                // RJCTODO: Need to let sucessfully started data source ingest 
-                // jobs know they should shut down.
                 break;
             }
         }
+
+        /**
+         * TODO: Need to let successfully started data source ingest jobs know
+         * they should shut down. This means that the start up of the ingest
+         * module pipelines and the submission of ingest tasks should be
+         * separated. This cancellation is just a stop gap; fortunately, if
+         * startup is going to fail, it will likely fail for the first child
+         * data source ingest job.
+         */
+        if (!errors.isEmpty()) {
+            for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) {
+                dataSourceJob.cancel();
+            }
+        }
+
         return errors;
     }
 
@@ -106,26 +123,8 @@ public final class IngestJob {
      *
      * @return The snapshot.
      */
-    synchronized public ProgressSnapshot getSnapshot() {
+    public ProgressSnapshot getSnapshot() {
-        DataSourceIngestModuleHandle moduleHandle = null;
+        return new ProgressSnapshot();
-        boolean fileIngestIsRunning = false;
-        Date fileIngestStartTime = null;
-        for (DataSourceIngestJob.Snapshot snapshot : this.getDataSourceIngestJobSnapshots()) {
-            if (null == moduleHandle) {
-                DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule();
-                if (null != module) {
-                    moduleHandle = new DataSourceIngestModuleHandle(this.dataSourceJobs.get(snapshot.getJobId()), module);
-                }
-            }
-            if (snapshot.fileIngestIsRunning()) {
-                fileIngestIsRunning = true;
-            }
-            Date childFileIngestStartTime = snapshot.fileIngestStartTime();
-            if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) {
-                fileIngestStartTime = childFileIngestStartTime;
-            }
-        }
-        return new ProgressSnapshot(moduleHandle, fileIngestIsRunning, fileIngestStartTime, this.cancelled);
     }
 
     /**
@@ -134,7 +133,7 @@ public final class IngestJob {
      *
      * @return A list of data source ingest job progress snapshots.
      */
-    synchronized List<DataSourceIngestJob.Snapshot> getDataSourceIngestJobSnapshots() {
+    List<DataSourceIngestJob.Snapshot> getDataSourceIngestJobSnapshots() {
         List<DataSourceIngestJob.Snapshot> snapshots = new ArrayList<>();
         for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) {
             snapshots.add(dataSourceJob.getSnapshot());
@@ -148,7 +147,7 @@ public final class IngestJob {
      * but there may be a delay before all of the ingest modules in the
      * pipelines respond by stopping processing.
      */
-    synchronized public void cancel() {
+    public void cancel() {
         for (DataSourceIngestJob job : this.dataSourceJobs.values()) {
             job.cancel();
         }
@@ -161,7 +160,7 @@ public final class IngestJob {
      *
      * @return True or false.
      */
-    synchronized public boolean isCancelled() {
+    public boolean isCancelled() {
         return this.cancelled;
     }
 
@@ -171,9 +170,8 @@ public final class IngestJob {
      *
      * @param dataSourceIngestJob A completed data source ingest job.
      */
-    synchronized void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) {
+    void dataSourceJobFinished(DataSourceIngestJob dataSourceIngestJob) {
-        this.dataSourceJobs.remove(dataSourceIngestJob.getId());
+        if (incompleteJobsCount.decrementAndGet() == 0) {
-        if (this.dataSourceJobs.isEmpty()) {
             IngestManager.getInstance().finishIngestJob(this);
         }
     }
@@ -181,30 +179,85 @@ public final class IngestJob {
     /**
      * A snapshot of the progress of an ingest job.
      */
-    public static final class ProgressSnapshot {
+    public final class ProgressSnapshot {
 
-        private final DataSourceIngestModuleHandle dataSourceModule;
+        private final List<DataSourceProcessingSnapshot> dataSourceProcessingSnapshots;
-        private final boolean fileIngestRunning;
+        private DataSourceIngestModuleHandle dataSourceModule;
-        private final Date fileIngestStartTime;
+        private boolean fileIngestRunning;
-        private final boolean cancelled;
+        private Date fileIngestStartTime;
+        private final boolean jobCancelled;
+
+        /**
+         * A snapshot of the progress of an ingest job on the processing of a
+         * data source.
+         */
+        public final class DataSourceProcessingSnapshot {
+
+            private final DataSourceIngestJob.Snapshot snapshot;
+
+            private DataSourceProcessingSnapshot(DataSourceIngestJob.Snapshot snapshot) {
+                this.snapshot = snapshot;
+            }
+
+            /**
+             * Gets the name of the data source that is the subject of this
+             * snapshot.
+             *
+             * @return A data source name string.
+             */
+            public String getDataSource() {
+                return snapshot.getDataSource();
+            }
+
+            /**
+             * Indicates whether or not the processing of the data source that
+             * is the subject of this snapshot was canceled.
+             *
+             * @return True or false.
+             */
+            public boolean isCancelled() {
+                return snapshot.isCancelled();
+            }
+
+            /**
+             * Gets a list of the display names of any canceled data source
+             * level ingest modules.
+             *
+             * @return A list of canceled data source level ingest module
+             * display names, possibly empty.
+             */
+            public List<String> getCancelledDataSourceIngestModules() {
+                return snapshot.getCancelledDataSourceIngestModules();
+            }
+
+        }
 
         /**
          * Constructs a snapshot of ingest job progress.
-         *
-         * @param dataSourceModule The currently running data source level
-         * ingest module, may be null.
-         * @param fileIngestRunning Whether or not file ingest is currently
-         * running.
-         * @param fileIngestStartTime The start time of file level ingest, may
-         * be null.
-         * @param cancelled Whether or not a cancellation request has been
-         * issued.
          */
-        private ProgressSnapshot(DataSourceIngestModuleHandle dataSourceModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean cancelled) {
+        private ProgressSnapshot() {
-            this.dataSourceModule = dataSourceModule;
+            dataSourceModule = null;
-            this.fileIngestRunning = fileIngestRunning;
+            fileIngestRunning = false;
-            this.fileIngestStartTime = fileIngestStartTime;
+            fileIngestStartTime = null;
-            this.cancelled = cancelled;
+            dataSourceProcessingSnapshots = new ArrayList<>();
+            for (DataSourceIngestJob dataSourceJob : dataSourceJobs.values()) {
+                DataSourceIngestJob.Snapshot snapshot = dataSourceJob.getSnapshot();
+                dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot));
+                if (null == dataSourceModule) {
+                    DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule();
+                    if (null != module) {
+                        dataSourceModule = new DataSourceIngestModuleHandle(dataSourceJobs.get(snapshot.getJobId()), module);
+                    }
+                }
+                if (snapshot.fileIngestIsRunning()) {
+                    fileIngestRunning = true;
+                }
+                Date childFileIngestStartTime = snapshot.fileIngestStartTime();
+                if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) {
+                    fileIngestStartTime = childFileIngestStartTime;
+                }
+            }
+            this.jobCancelled = cancelled;
         }
 
         /**
@@ -243,7 +296,16 @@ public final class IngestJob {
          * @return True or false.
          */
         public boolean isCancelled() {
-            return this.cancelled;
+            return this.jobCancelled;
+        }
+
+        /**
+         * Gets snapshots of the progress processing individual data sources.
+         *
+         * @return The list of snapshots.
+         */
+        public List<DataSourceProcessingSnapshot> getDataSourceSnapshots() {
+            return Collections.unmodifiableList(this.dataSourceProcessingSnapshots);
         }
 
     }

Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java

@@ -404,7 +404,7 @@ public class IngestManager {
         boolean success = false;
 
         if (this.jobCreationIsEnabled) {
-            if (runInteractively && jobsById.isEmpty()) { // RJCTODO: This is sort of broken
+            if (runInteractively && jobsById.isEmpty()) {
                 clearIngestMessageBox();
             }
 

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java

@@ -149,7 +149,7 @@ class QueryResults {
                     if (writeResult != null) {
                         newArtifacts.add(writeResult.getArtifact());
                         if (notifyInbox) {
-                            writeSingleFileInboxMessage(writeResult, hit.getContent()); // RJCTODO: Consider rewriting this message post code 
+                            writeSingleFileInboxMessage(writeResult, hit.getContent()); 
                         }
                     } else {
                         logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hit.getContent(), keyword.toString()}); //NON-NLS

ScalpelCarver/nbproject/platform.properties

@@ -0,0 +1,123 @@
+branding.token=autopsy
+# Version of platform that is automatically downloaded
+# Note build.xml has similar definitions that should be kept in sync (manually)
+netbeans-plat-version=7.3.1
+suite.dir=${basedir}
+nbplatform.active.dir=${suite.dir}/netbeans-plat/${netbeans-plat-version}
+harness.dir=${nbplatform.active.dir}/harness
+bootstrap.url=http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/netbeans/harness/tasks.jar
+# Where we get the platform from. To see what versions are available, open URL in browser up to the .../updates part of the URL
+autoupdate.catalog.url=http://dlc.sun.com.edgesuite.net/netbeans/updates/${netbeans-plat-version}/uc/final/distribution/catalog.xml.gz
+cluster.path=\
+    ${nbplatform.active.dir}/harness:\
+    ${nbplatform.active.dir}/java:\
+    ${nbplatform.active.dir}/platform
+disabled.modules=\
+    org.apache.tools.ant.module,\
+    org.netbeans.api.debugger.jpda,\
+    org.netbeans.api.java,\
+    org.netbeans.lib.nbjavac,\
+    org.netbeans.libs.cglib,\
+    org.netbeans.libs.javacapi,\
+    org.netbeans.libs.javacimpl,\
+    org.netbeans.libs.springframework,\
+    org.netbeans.modules.ant.browsetask,\
+    org.netbeans.modules.ant.debugger,\
+    org.netbeans.modules.ant.freeform,\
+    org.netbeans.modules.ant.grammar,\
+    org.netbeans.modules.ant.kit,\
+    org.netbeans.modules.beans,\
+    org.netbeans.modules.classfile,\
+    org.netbeans.modules.dbschema,\
+    org.netbeans.modules.debugger.jpda,\
+    org.netbeans.modules.debugger.jpda.ant,\
+    org.netbeans.modules.debugger.jpda.kit,\
+    org.netbeans.modules.debugger.jpda.projects,\
+    org.netbeans.modules.debugger.jpda.ui,\
+    org.netbeans.modules.debugger.jpda.visual,\
+    org.netbeans.modules.findbugs.installer,\
+    org.netbeans.modules.form,\
+    org.netbeans.modules.form.binding,\
+    org.netbeans.modules.form.j2ee,\
+    org.netbeans.modules.form.kit,\
+    org.netbeans.modules.form.nb,\
+    org.netbeans.modules.form.refactoring,\
+    org.netbeans.modules.hibernate,\
+    org.netbeans.modules.hibernatelib,\
+    org.netbeans.modules.hudson.ant,\
+    org.netbeans.modules.hudson.maven,\
+    org.netbeans.modules.i18n,\
+    org.netbeans.modules.i18n.form,\
+    org.netbeans.modules.j2ee.core.utilities,\
+    org.netbeans.modules.j2ee.eclipselink,\
+    org.netbeans.modules.j2ee.eclipselinkmodelgen,\
+    org.netbeans.modules.j2ee.jpa.refactoring,\
+    org.netbeans.modules.j2ee.jpa.verification,\
+    org.netbeans.modules.j2ee.metadata,\
+    org.netbeans.modules.j2ee.metadata.model.support,\
+    org.netbeans.modules.j2ee.persistence,\
+    org.netbeans.modules.j2ee.persistence.kit,\
+    org.netbeans.modules.j2ee.persistenceapi,\
+    org.netbeans.modules.java.api.common,\
+    org.netbeans.modules.java.debug,\
+    org.netbeans.modules.java.editor,\
+    org.netbeans.modules.java.editor.lib,\
+    org.netbeans.modules.java.examples,\
+    org.netbeans.modules.java.freeform,\
+    org.netbeans.modules.java.guards,\
+    org.netbeans.modules.java.helpset,\
+    org.netbeans.modules.java.hints,\
+    org.netbeans.modules.java.hints.declarative,\
+    org.netbeans.modules.java.hints.declarative.test,\
+    org.netbeans.modules.java.hints.legacy.spi,\
+    org.netbeans.modules.java.hints.test,\
+    org.netbeans.modules.java.hints.ui,\
+    org.netbeans.modules.java.j2seplatform,\
+    org.netbeans.modules.java.j2seproject,\
+    org.netbeans.modules.java.kit,\
+    org.netbeans.modules.java.lexer,\
+    org.netbeans.modules.java.navigation,\
+    org.netbeans.modules.java.platform,\
+    org.netbeans.modules.java.preprocessorbridge,\
+    org.netbeans.modules.java.project,\
+    org.netbeans.modules.java.source,\
+    org.netbeans.modules.java.source.ant,\
+    org.netbeans.modules.java.source.queries,\
+    org.netbeans.modules.java.source.queriesimpl,\
+    org.netbeans.modules.java.sourceui,\
+    org.netbeans.modules.java.testrunner,\
+    org.netbeans.modules.javadoc,\
+    org.netbeans.modules.javawebstart,\
+    org.netbeans.modules.junit,\
+    org.netbeans.modules.maven,\
+    org.netbeans.modules.maven.checkstyle,\
+    org.netbeans.modules.maven.coverage,\
+    org.netbeans.modules.maven.embedder,\
+    org.netbeans.modules.maven.grammar,\
+    org.netbeans.modules.maven.graph,\
+    org.netbeans.modules.maven.hints,\
+    org.netbeans.modules.maven.indexer,\
+    org.netbeans.modules.maven.junit,\
+    org.netbeans.modules.maven.kit,\
+    org.netbeans.modules.maven.model,\
+    org.netbeans.modules.maven.osgi,\
+    org.netbeans.modules.maven.persistence,\
+    org.netbeans.modules.maven.refactoring,\
+    org.netbeans.modules.maven.repository,\
+    org.netbeans.modules.maven.search,\
+    org.netbeans.modules.maven.spring,\
+    org.netbeans.modules.projectimport.eclipse.core,\
+    org.netbeans.modules.projectimport.eclipse.j2se,\
+    org.netbeans.modules.refactoring.java,\
+    org.netbeans.modules.spellchecker.bindings.java,\
+    org.netbeans.modules.spring.beans,\
+    org.netbeans.modules.testng,\
+    org.netbeans.modules.testng.ant,\
+    org.netbeans.modules.testng.maven,\
+    org.netbeans.modules.websvc.jaxws21,\
+    org.netbeans.modules.websvc.jaxws21api,\
+    org.netbeans.modules.websvc.saas.codegen.java,\
+    org.netbeans.modules.xml.jaxb,\
+    org.netbeans.modules.xml.tools.java,\
+    org.netbeans.spi.java.hints
+

ScalpelCarver/nbproject/project.xml

@@ -4,7 +4,7 @@
     <configuration>
         <data xmlns="http://www.netbeans.org/ns/nb-module-project/3">
             <code-name-base>org.sleuthkit.autopsy.scalpel</code-name-base>
-            <suite-component/>
+            <standalone/>
             <module-dependencies>
                 <dependency>
                     <code-name-base>org.openide.util</code-name-base>

ScalpelCarver/nbproject/suite.properties

@@ -1 +0,0 @@
-suite.dir=${basedir}/..

nbproject/project.properties

@@ -27,13 +27,11 @@ modules=\
     ${project.org.sleuthkit.autopsy.testing}:\
     ${project.org.sleuthkit.autopsy.thunderbirdparser}:\
     ${project.org.sleuthkit.autopsy.core}:\
-    ${project.org.sleuthkit.autopsy.corelibs}:\
+    ${project.org.sleuthkit.autopsy.corelibs}
-    ${project.org.sleuthkit.autopsy.scalpel}
 project.org.sleuthkit.autopsy.core=Core
 project.org.sleuthkit.autopsy.corelibs=CoreLibs
 project.org.sleuthkit.autopsy.keywordsearch=KeywordSearch
 project.org.sleuthkit.autopsy.recentactivity=RecentActivity
 project.org.sleuthkit.autopsy.testing=Testing
 project.org.sleuthkit.autopsy.thunderbirdparser=thunderbirdparser
-project.org.sleuthkit.autopsy.scalpel=ScalpelCarver