From 9a296e2af665030694574b1036566cf2ed22e840 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 25 Feb 2021 12:34:08 -0500 Subject: [PATCH 01/63] 7332 Add data artifact ingest pipeline --- .../ingest/DataArtifactIngestModule.java | 38 +++++ .../ingest/DataArtifactIngestPipeline.java | 97 +++++++++++ .../ingest/DataArtifactIngestTask.java | 68 ++++++++ .../autopsy/ingest/DataSourceIngestTask.java | 36 ++++- .../autopsy/ingest/FileIngestPipeline.java | 32 ++++ .../autopsy/ingest/FileIngestTask.java | 52 ++++-- .../autopsy/ingest/IngestManager.java | 11 +- .../autopsy/ingest/IngestModule.java | 2 +- .../autopsy/ingest/IngestModuleFactory.java | 89 ++++++++-- .../ingest/IngestModuleFactoryAdapter.java | 6 +- .../sleuthkit/autopsy/ingest/IngestTask.java | 66 +++++++- .../autopsy/ingest/IngestTasksScheduler.java | 153 +++++++++++------- 12 files changed, 557 insertions(+), 93 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java new file mode 100755 index 0000000000..f45654185c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java @@ -0,0 +1,38 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.datamodel.DataArtifact; + +/** + * Interface that must be implemented by all ingest modules that process data + * artifacts. + */ +public interface DataArtifactIngestModule extends IngestModule { + + /** + * Processes a data artifact. + * + * @param artifact The artifact to process. + * + * @return A result code indicating success or failure of the processing. + */ + ProcessResult process(DataArtifact artifact); + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java new file mode 100755 index 0000000000..cdd5d7937e --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java @@ -0,0 +1,97 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.List; +import java.util.Optional; + +/** + * A pipeline of data artifact ingest modules for performing data artifact + * ingest tasks for an ingest job. + */ +final class DataArtifactIngestPipeline extends IngestTaskPipeline { + + private final IngestJobPipeline ingestJobPipeline; + + /** + * Constructs a pipeline of data artifact ingest modules for performing data + * artifact ingest tasks for an ingest job. + * + * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. + * @param moduleTemplates The ingest module templates that define this + * pipeline. + */ + DataArtifactIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + super(ingestJobPipeline, moduleTemplates); + this.ingestJobPipeline = ingestJobPipeline; + } + + @Override + Optional> acceptModuleTemplate(IngestModuleTemplate ingestModuleTemplate) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + void prepareTask(DataArtifactIngestTask task) throws IngestTaskPipelineException { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + void completeTask(DataArtifactIngestTask task) throws IngestTaskPipelineException { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + /** + * A wrapper that adds ingest infrastructure operations to a data artifact + * ingest module. + */ + static final class DataArtifactIngestPipelineModule extends IngestTaskPipeline.PipelineModule { + + private final DataArtifactIngestModule module; + + /** + * Constructs a wrapper that adds ingest infrastructure operations to a + * file ingest module. + * + * + * @param module The module. + * @param displayName The display name of the module. + */ + DataArtifactIngestPipelineModule(DataArtifactIngestModule module, String displayName) { + super(module, displayName); + this.module = module; + } + + /** + * RJCTODO + * + * @param ingestJobPipeline + * @param task + * + * @throws + * org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + */ + @Override + void performTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModule.IngestModuleException { + // RJCTODO: Fill in and change to executeTask() + } + + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java new file mode 100755 index 0000000000..e3fbf6a688 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java @@ -0,0 +1,68 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.datamodel.DataArtifact; + +/** + * A data artifact ingest task that will be executed by an ingest thread using a + * given ingest job pipeline. + */ +final class DataArtifactIngestTask extends IngestTask { + + private final DataArtifact artifact; + + /** + * Constructs a data artifact ingest task that will be executed by an ingest + * thread using a given ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline to use to complete the + * task. + * @param artifact The data artifact to be processed. + */ + DataArtifactIngestTask(IngestJobPipeline ingestJobPipeline, DataArtifact artifact) { + super(ingestJobPipeline); + this.artifact = artifact; + } + + /** + * Gets the data artifact for this task. + * + * @return The data artifact. + */ + DataArtifact getDataArtifact() { + return artifact; + } + + /** + * Executes this task by passing it to the given ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline. + * + * @throws InterruptedException This exception is thrown if the thread + * executing the task is interrupted while + * blocked. + */ + @Override + void execute(IngestJobPipeline ingestJobPipeline) throws InterruptedException { + // RJCTODO: Need overload for data artifacts + // ingestJobPipeline.process(this); + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index 417b7bee96..dd7e217044 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2012-2014 Basis Technology Corp. + * + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,15 +18,35 @@ */ package org.sleuthkit.autopsy.ingest; +/** + * A data source level ingest task that will be executed by an ingest thread + * using a given ingest job pipeline. + */ final class DataSourceIngestTask extends IngestTask { + /** + * Constructs a data source level ingest task that will be executed by an + * ingest thread using a given ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline to use to complete the + * task. + */ DataSourceIngestTask(IngestJobPipeline ingestJobPipeline) { super(ingestJobPipeline); } + /** + * Executes this task by passing it to the given ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline. + * + * @throws InterruptedException This exception is thrown if the thread + * executing the task is interrupted while + * blocked. + */ @Override - void execute(long threadId) throws InterruptedException { - super.setThreadId(threadId); - getIngestJobPipeline().process(this); + void execute(IngestJobPipeline ingestJobPipeline) throws InterruptedException { + ingestJobPipeline.process(this); } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 1cc6ce05a4..2c1142534e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -45,6 +45,13 @@ final class FileIngestPipeline extends IngestTaskPipeline { this.ingestJobPipeline = ingestJobPipeline; } + /** + * RJCTODO + * + * @param template + * + * @return + */ @Override Optional> acceptModuleTemplate(IngestModuleTemplate template) { Optional> module = Optional.empty(); @@ -55,10 +62,26 @@ final class FileIngestPipeline extends IngestTaskPipeline { return module; } + /** + * RJCTODO + * + * @param task + * + * @throws + * org.sleuthkit.autopsy.ingest.IngestTaskPipeline.IngestTaskPipelineException + */ @Override void prepareTask(FileIngestTask task) throws IngestTaskPipelineException { } + /** + * RJCTODO + * + * @param task + * + * @throws + * org.sleuthkit.autopsy.ingest.IngestTaskPipeline.IngestTaskPipelineException + */ @Override void completeTask(FileIngestTask task) throws IngestTaskPipelineException { AbstractFile file = null; @@ -107,6 +130,15 @@ final class FileIngestPipeline extends IngestTaskPipeline { this.module = module; } + /** + * RJCTODO + * + * @param ingestJobPipeline + * @param task + * + * @throws + * org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + */ @Override void performTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException { AbstractFile file = null; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 8bb918cce1..067523f7c1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2012-2015 Basis Technology Corp. + * + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,14 +24,22 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * Represents a single file analysis task, which is defined by a file to analyze - * and the InjestJob/Pipeline to run it on. + * A file ingest task that will be executed by an ingest thread using a given + * ingest job pipeline. */ final class FileIngestTask extends IngestTask { - + private final long fileId; private AbstractFile file = null; + /** + * Constructs a file ingest task that will be executed by an ingest thread + * using a given ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline to use to complete the + * task. + * @param file The file to be processed. + */ FileIngestTask(IngestJobPipeline ingestJobPipeline, AbstractFile file) { super(ingestJobPipeline); this.file = file; @@ -43,10 +51,23 @@ final class FileIngestTask extends IngestTask { this.fileId = fileId; } + /** + * Gets the object ID of the file for this task. + * + * @return The object ID. + */ long getFileId() { return fileId; } + /** + * Gets the file for this task. + * + * @return The file. + * + * @throws TskCoreException The exception is thrown if there is an error + * retieving the file from the case database. + */ synchronized AbstractFile getFile() throws TskCoreException { if (file == null) { file = Case.getCurrentCase().getSleuthkitCase().getAbstractFileById(fileId); @@ -54,10 +75,18 @@ final class FileIngestTask extends IngestTask { return file; } + /** + * Executes this task by passing it to the given ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline. + * + * @throws InterruptedException This exception is thrown if the thread + * executing the task is interrupted while + * blocked. + */ @Override - void execute(long threadId) throws InterruptedException { - super.setThreadId(threadId); - getIngestJobPipeline().process(this); + void execute(IngestJobPipeline ingestJobPipeline) throws InterruptedException { + ingestJobPipeline.process(this); } @Override @@ -84,4 +113,5 @@ final class FileIngestTask extends IngestTask { hash = 47 * hash + Objects.hashCode(this.fileId); return hash; } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 791a45021c..0a0fb12f07 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -125,8 +125,9 @@ public class IngestManager implements IngestProgressSnapshotProvider { private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-start-ingest-jobs-%d").build()); //NON-NLS; private final Map> startIngestJobFutures = new ConcurrentHashMap<>(); private final Map ingestJobsById = new HashMap<>(); - private final ExecutorService dataSourceLevelIngestJobTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS; + private final ExecutorService dataSourceLevelIngestJobTasksExecutor; private final ExecutorService fileLevelIngestJobTasksExecutor; + private final ExecutorService dataArtifactIngestTasksExecutor; private final ExecutorService eventPublishingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-ingest-events-%d").build()); //NON-NLS; private final IngestMonitor ingestMonitor = new IngestMonitor(); private final ServicesMonitor servicesMonitor = ServicesMonitor.getInstance(); @@ -164,6 +165,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { * source level ingest job tasks to the data source level ingest job * tasks executor. */ + dataSourceLevelIngestJobTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS; long threadId = nextIngestManagerTaskId.incrementAndGet(); dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); @@ -180,6 +182,13 @@ public class IngestManager implements IngestProgressSnapshotProvider { fileLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } + + dataArtifactIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-artifact-ingest-%d").build()); //NON-NLS; + threadId = nextIngestManagerTaskId.incrementAndGet(); + // RJCTODO + // dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); + // ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + // RJCTODO: Where is the shut down code? } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java index 8bfeef57fb..01e771742a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java @@ -67,7 +67,7 @@ public interface IngestModule { */ default void startUp(IngestJobContext context) throws IngestModuleException { } - + /** * Invoked by Autopsy when an ingest job is completed (either because the * data has been analyzed or because the job was cancelled), before the diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java index 5b522d552a..b0ac69756d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -98,7 +98,9 @@ public interface IngestModuleFactory { * * @return True if the factory provides a global settings panel. */ - boolean hasGlobalSettingsPanel(); + default boolean hasGlobalSettingsPanel() { + return false; + } /** * Gets a user interface panel that allows a user to change settings that @@ -113,7 +115,9 @@ public interface IngestModuleFactory { * * @return A global settings panel. */ - IngestModuleGlobalSettingsPanel getGlobalSettingsPanel(); + default IngestModuleGlobalSettingsPanel getGlobalSettingsPanel() { + throw new UnsupportedOperationException(); + } /** * Gets the default per ingest job settings for instances of the family of @@ -127,7 +131,9 @@ public interface IngestModuleFactory { * * @return The default ingest job settings. */ - IngestModuleIngestJobSettings getDefaultIngestJobSettings(); + default IngestModuleIngestJobSettings getDefaultIngestJobSettings() { + return new NoIngestModuleIngestJobSettings(); + } /** * Queries the factory to determine if it provides user a interface panel to @@ -141,7 +147,9 @@ public interface IngestModuleFactory { * * @return True if the factory provides ingest job settings panels. */ - boolean hasIngestJobSettingsPanel(); + default boolean hasIngestJobSettingsPanel() { + return false; + } /** * Gets a user interface panel that can be used to set per ingest job @@ -157,7 +165,9 @@ public interface IngestModuleFactory { * * @return An ingest job settings panel. */ - IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings); + default IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { + throw new UnsupportedOperationException(); + } /** * Queries the factory to determine if it is capable of creating data source @@ -167,7 +177,9 @@ public interface IngestModuleFactory { * * @return True if the factory can create data source ingest modules. */ - boolean isDataSourceIngestModuleFactory(); + default boolean isDataSourceIngestModuleFactory() { + return false; + } /** * Creates a data source ingest module instance. @@ -198,7 +210,9 @@ public interface IngestModuleFactory { * * @return A data source ingest module instance. */ - DataSourceIngestModule createDataSourceIngestModule(IngestModuleIngestJobSettings settings); + default DataSourceIngestModule createDataSourceIngestModule(IngestModuleIngestJobSettings ingestOptions) { + throw new UnsupportedOperationException(); + } /** * Queries the factory to determine if it is capable of creating file ingest @@ -208,7 +222,9 @@ public interface IngestModuleFactory { * * @return True if the factory can create file ingest modules. */ - boolean isFileIngestModuleFactory(); + default boolean isFileIngestModuleFactory() { + return false; + } /** * Creates a file ingest module instance. @@ -238,6 +254,59 @@ public interface IngestModuleFactory { * @param settings The settings for the ingest job. * * @return A file ingest module instance. + * RJCTODO: Remove refs to adapters */ - FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings); + default FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { + throw new UnsupportedOperationException(); + } + + /** + * RJCTODO + * + * Queries the factory to determine if it is capable of creating file ingest + * modules. If the module family does not include file ingest modules, the + * factory may extend IngestModuleFactoryAdapter to get an implementation of + * this method that returns false. + * + * @return True if the factory can create file ingest modules. + */ + default boolean isDataArtifactIngestModuleFactory() { + return false; + } + + /** + * RJCTODO + * + * Creates a file ingest module instance. + *

+ * Autopsy will generally use the factory to several instances of each type + * of module for each ingest job it performs. Completing an ingest job + * entails processing a single data source (e.g., a disk image) and all of + * the files from the data source, including files extracted from archives + * and any unallocated space (made to look like a series of files). The data + * source is passed through one or more pipelines of data source ingest + * modules. The files are passed through one or more pipelines of file + * ingest modules. + *

+ * The ingest framework may use multiple threads to complete an ingest job, + * but it is guaranteed that there will be no more than one module instance + * per thread. However, if the module instances must share resources, the + * modules are responsible for synchronizing access to the shared resources + * and doing reference counting as required to release those resources + * correctly. Also, more than one ingest job may be in progress at any given + * time. This must also be taken into consideration when sharing resources + * between module instances. modules. + *

+ * If the module family does not include file ingest modules, the factory + * may extend IngestModuleFactoryAdapter to get an implementation of this + * method that throws an UnsupportedOperationException. + * + * @param settings The settings for the ingest job. + * + * @return A file ingest module instance. + */ + default DataArtifactIngestModule createDataArtifactIngestModule(IngestModuleIngestJobSettings settings) { + throw new UnsupportedOperationException(); + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java index 46a7427f02..f376378235 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,10 @@ package org.sleuthkit.autopsy.ingest; /** * An adapter that provides no-op implementations of various IngestModuleFactory * methods. + * + * NOTE: As of Java 8, interfaces can have default methods. IngestModuleFactory + * now provides default no-op versions of all of its optional methods. This + * class is no longer needed and can be deprecated when convenient. */ public abstract class IngestModuleFactoryAdapter implements IngestModuleFactory { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index ebfabf2ab2..261edeb24f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2014 Basis Technology Corp. + * + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,32 +20,84 @@ package org.sleuthkit.autopsy.ingest; import org.sleuthkit.datamodel.Content; +/** + * An ingest task that will be executed by an ingest thread using a given ingest + * job pipeline. The task is one unit of analysis work for an ingest job for a + * data source. Some examples of concrete types of ingest tasks include the + * analysis of files and the analysis of data artifacts. + */ abstract class IngestTask { private final static long NOT_SET = Long.MIN_VALUE; private final IngestJobPipeline ingestJobPipeline; private long threadId; + /** + * Constructs an ingest task that will be executed by an ingest thread using + * a given ingest job pipeline. The task is one unit of analysis work for an + * ingest job for a data source. Some examples of concrete types of ingest + * tasks include the analysis of files and the analysis of data artifacts. + * + * @param ingestJobPipeline The ingest job pipeline to use to complete the + * task. + */ IngestTask(IngestJobPipeline ingestJobPipeline) { this.ingestJobPipeline = ingestJobPipeline; threadId = NOT_SET; } + /** + * Gets the ingest job pipeline used to complete this task. + * + * @return The ingest job pipeline. + */ IngestJobPipeline getIngestJobPipeline() { return ingestJobPipeline; } + /** + * Gets the data source for the ingest job of which this task is a part. + * + * @return The data source. + */ Content getDataSource() { return getIngestJobPipeline().getDataSource(); } + /** + * Gets the thread ID of the ingest thread executing this task. + * + * @return The thread ID. + */ long getThreadId() { return threadId; } - void setThreadId(long threadId) { + /** + * Records the ingest thread ID of the calling thread and executes this task + * using the ingest job pipeline specified when the task was created. + * + * @param threadId The numeric ID of the ingest thread executing this task. + * + * @throws InterruptedException This exception is thrown if the calling + * thread is interrupted while blocked waiting + * for some task operation to complete. + */ + void execute(long threadId) throws InterruptedException { // RJCTODO: Why does htis block? this.threadId = threadId; + execute(ingestJobPipeline); } - abstract void execute(long threadId) throws InterruptedException; + /** + * Executes this task using the ingest job pipeline specified when the task + * was created. + * + * @param ingestJobPipeline The ingest job pipeline. + * + * @throws InterruptedException This exception is thrown if the thread + * executing the task is interrupted while + * blocked. + */ + abstract void execute(IngestJobPipeline IngestJobPipeline) throws InterruptedException; + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 2ec96915cc..fe581ca2d5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2018 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -39,6 +39,7 @@ import javax.annotation.concurrent.ThreadSafe; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataArtifact; import org.sleuthkit.datamodel.FileSystem; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -62,6 +63,7 @@ final class IngestTasksScheduler { @GuardedBy("this") private final Queue streamedTasksQueue; private final IngestTaskTrackingQueue fileIngestThreadsQueue; + private final IngestTaskTrackingQueue dataArtifactIngestThreadQueue; /** * Gets the ingest tasks scheduler singleton that creates ingest tasks for @@ -81,11 +83,12 @@ final class IngestTasksScheduler { * the ingest manager's ingest threads. */ private IngestTasksScheduler() { - this.dataSourceIngestThreadQueue = new IngestTaskTrackingQueue(); - this.rootFileTaskQueue = new TreeSet<>(new RootDirectoryTaskComparator()); - this.pendingFileTaskQueue = new LinkedList<>(); - this.fileIngestThreadsQueue = new IngestTaskTrackingQueue(); - this.streamedTasksQueue = new LinkedList<>(); + dataSourceIngestThreadQueue = new IngestTaskTrackingQueue(); + rootFileTaskQueue = new TreeSet<>(new RootDirectoryTaskComparator()); + pendingFileTaskQueue = new LinkedList<>(); + fileIngestThreadsQueue = new IngestTaskTrackingQueue(); + streamedTasksQueue = new LinkedList<>(); + dataArtifactIngestThreadQueue = new IngestTaskTrackingQueue(); } /** @@ -95,7 +98,7 @@ final class IngestTasksScheduler { * @return The queue. */ BlockingIngestTaskQueue getDataSourceIngestTaskQueue() { - return this.dataSourceIngestThreadQueue; + return dataSourceIngestThreadQueue; } /** @@ -105,11 +108,21 @@ final class IngestTasksScheduler { * @return The queue. */ BlockingIngestTaskQueue getFileIngestTaskQueue() { - return this.fileIngestThreadsQueue; + return fileIngestThreadsQueue; } /** - * Schedules a data source level ingest task and zero to many file level + * Gets the data artifacts ingest tasks queue. This queue is a blocking + * queue used by the ingest manager's data artifact ingest thread. + * + * @return The queue. + */ + BlockingIngestTaskQueue getDataArtifactIngestTaskQueue() { + return dataArtifactIngestThreadQueue; + } + + /** + * Schedules a data source level ingest task and zero to many file level * ingest tasks for an ingest job pipeline. * * @param ingestJobPipeline The ingest job pipeline. @@ -118,11 +131,11 @@ final class IngestTasksScheduler { if (!ingestJobPipeline.isCancelled()) { /* * Scheduling of both the data source ingest task and the initial - * file ingest tasks for an ingestJobPipeline must be an atomic operation. - * Otherwise, the data source task might be completed before the - * file tasks are scheduled, resulting in a potential false positive - * when another thread checks whether or not all the tasks for the - * ingestJobPipeline are completed. + * file ingest tasks for an ingestJobPipeline must be an atomic + * operation. Otherwise, the data source task might be completed + * before the file tasks are scheduled, resulting in a potential + * false positive when another thread checks whether or not all the + * tasks for the ingestJobPipeline are completed. */ this.scheduleDataSourceIngestTask(ingestJobPipeline); this.scheduleFileIngestTasks(ingestJobPipeline, Collections.emptyList()); @@ -150,9 +163,10 @@ final class IngestTasksScheduler { * Schedules file tasks for either all the files or a given subset of the * files for an ingest job pipeline. * - * @param ingestJobPipeline The ingest job pipeline. - * @param files A subset of the files for the data source; if empty, then - * file tasks for all files in the data source are scheduled. + * @param ingestJobPipeline The ingest job pipeline. + * @param files A subset of the files for the data source; if + * empty, then file tasks for all files in the data + * source are scheduled. */ synchronized void scheduleFileIngestTasks(IngestJobPipeline ingestJobPipeline, Collection files) { if (!ingestJobPipeline.isCancelled()) { @@ -171,33 +185,34 @@ final class IngestTasksScheduler { refillIngestThreadQueue(); } } - + /** * Schedules file tasks for the given list of file IDs. * - * @param ingestJobPipeline The ingest job pipeline. - * @param files A subset of the files for the data source; if empty, then - * file tasks for all files in the data source are scheduled. + * @param ingestJobPipeline The ingest job pipeline. + * @param files A subset of the files for the data source; if + * empty, then file tasks for all files in the data + * source are scheduled. */ synchronized void scheduleStreamedFileIngestTasks(IngestJobPipeline ingestJobPipeline, List fileIds) { if (!ingestJobPipeline.isCancelled()) { for (long id : fileIds) { - // Create the file ingest task. Note that we do not do the shouldEnqueueFileTask() - // check here in order to delay loading the AbstractFile object. + // Create the file ingest task. Note that we do not do the shouldEnqueueFileTask() + // check here in order to delay loading the AbstractFile object. FileIngestTask task = new FileIngestTask(ingestJobPipeline, id); this.streamedTasksQueue.add(task); } refillIngestThreadQueue(); } - } + } /** * Schedules file level ingest tasks for a given set of files for an ingest - * job pipeline by adding them directly to the front of the file tasks - * queue for the ingest manager's file ingest threads. + * job pipeline by adding them directly to the front of the file tasks queue + * for the ingest manager's file ingest threads. * - * @param ingestJobPipeline The ingestJobPipeline. - * @param files A set of files for the data source. + * @param ingestJobPipeline The ingestJobPipeline. + * @param files A set of files for the data source. */ synchronized void fastTrackFileIngestTasks(IngestJobPipeline ingestJobPipeline, Collection files) { if (!ingestJobPipeline.isCancelled()) { @@ -224,6 +239,25 @@ final class IngestTasksScheduler { } } + /** + * Schedules a data artifact ingest task for an ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline. + */ + synchronized void scheduleDataArtifactIngestTasks(IngestJobPipeline ingestJobPipeline, Collection artifacts) { + if (!ingestJobPipeline.isCancelled()) { + for (DataArtifact artifact : artifacts) { + DataArtifactIngestTask task = new DataArtifactIngestTask(ingestJobPipeline, artifact); + try { + this.dataArtifactIngestThreadQueue.putLast(task); + } catch (InterruptedException ex) { + IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data artifact ingest task queue (jobId={%d)", ingestJobPipeline.getId()), ex); + Thread.currentThread().interrupt(); + } + } + } + } + /** * Allows an ingest thread to notify this ingest task scheduler that a data * source level task has been completed. @@ -231,7 +265,7 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(DataSourceIngestTask task) { - this.dataSourceIngestThreadQueue.taskCompleted(task); + dataSourceIngestThreadQueue.taskCompleted(task); } /** @@ -241,7 +275,18 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(FileIngestTask task) { - this.fileIngestThreadsQueue.taskCompleted(task); + fileIngestThreadsQueue.taskCompleted(task); + refillIngestThreadQueue(); + } + + /** + * Allows an ingest thread to notify this ingest task scheduler that a data + * artifact ingest task has been completed. + * + * @param task The completed task. + */ + synchronized void notifyTaskCompleted(DataArtifactIngestTask task) { + dataArtifactIngestThreadQueue.taskCompleted(task); refillIngestThreadQueue(); } @@ -264,9 +309,9 @@ final class IngestTasksScheduler { } /** - * Clears the "upstream" task scheduling queues for an ingest pipeline, - * but does nothing about tasks that have already been moved into the - * queue that is consumed by the file ingest threads. + * Clears the "upstream" task scheduling queues for an ingest pipeline, but + * does nothing about tasks that have already been moved into the queue that + * is consumed by the file ingest threads. * * @param ingestJobPipeline The ingestJobPipeline. */ @@ -317,7 +362,7 @@ final class IngestTasksScheduler { } return topLevelFiles; } - + /** * Schedules file ingest tasks for the ingest manager's file ingest threads. * Files from streaming ingest will be prioritized. @@ -391,8 +436,8 @@ final class IngestTasksScheduler { * files derived from prioritized files. */ synchronized private void takeFromBatchTasksQueues() throws InterruptedException { - - while (this.fileIngestThreadsQueue.isEmpty()) { + + while (this.fileIngestThreadsQueue.isEmpty()) { /* * If the pending file task queue is empty, move the highest * priority root file task, if there is one, into it. @@ -414,11 +459,11 @@ final class IngestTasksScheduler { return; } if (shouldEnqueueFileTask(pendingTask)) { - /* - * The task is added to the queue for the ingest threads - * AFTER the higher priority tasks that preceded it. - */ - this.fileIngestThreadsQueue.putLast(pendingTask); + /* + * The task is added to the queue for the ingest threads AFTER + * the higher priority tasks that preceded it. + */ + this.fileIngestThreadsQueue.putLast(pendingTask); } /* @@ -544,11 +589,11 @@ final class IngestTasksScheduler { } /** - * Check whether or not a file should be carved for a data source ingest + * Check whether or not a file should be carved for a data source ingest * ingest job. - * + * * @param task The file ingest task for the file. - * + * * @return True or false. */ private static boolean shouldBeCarved(final FileIngestTask task) { @@ -561,12 +606,12 @@ final class IngestTasksScheduler { } /** - * Checks whether or not a file is accepted (passes) the file filter for a data - * source ingest job. + * Checks whether or not a file is accepted (passes) the file filter for a + * data source ingest job. * * @param task The file ingest task for the file. - * - * @return True or false. + * + * @return True or false. */ private static boolean fileAcceptedByFilter(final FileIngestTask task) { try { @@ -664,23 +709,23 @@ final class IngestTasksScheduler { } catch (TskCoreException ex) { // Do nothing - the exception has been logged elsewhere } - + try { file2 = q2.getFile(); } catch (TskCoreException ex) { // Do nothing - the exception has been logged elsewhere } - + if (file1 == null) { if (file2 == null) { return (int) (q2.getFileId() - q1.getFileId()); } else { return 1; } - } else if (file2 == null) { + } else if (file2 == null) { return -1; } - + AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(file1); AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(file2); if (p1 == p2) { @@ -956,7 +1001,7 @@ final class IngestTasksScheduler { * * @param jobId The identifier associated with the job. */ - IngestJobTasksSnapshot(long jobId, long dsQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, + IngestJobTasksSnapshot(long jobId, long dsQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, long runningListSize, long streamingQueueSize) { this.jobId = jobId; this.dsQueueSize = dsQueueSize; @@ -1000,7 +1045,7 @@ final class IngestTasksScheduler { long getFileQueueSize() { return fileQueueSize; } - + long getStreamingQueueSize() { return streamingQueueSize; } From cca59c02beb7073f2fb2edfa6815b5eab5d6f143 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 25 Feb 2021 12:37:17 -0500 Subject: [PATCH 02/63] 7332 Add data artifact ingest pipeline --- .../sleuthkit/autopsy/ingest/IngestJobPipeline.java | 12 +++++++++++- .../autopsy/ingest/IngestTasksScheduler.java | 10 ++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 384f5c63c5..1a97b9e161 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -52,6 +52,7 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.modules.interestingitems.FilesSet; import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer; +import org.sleuthkit.datamodel.DataArtifact; /** * Encapsulates a data source and the ingest module pipelines used to process @@ -90,7 +91,7 @@ final class IngestJobPipeline { */ INITIALIZATION, /** - * Running only file ingest modules (used only for streaming ingest) + * Running only file ingest modules (used only for streaming ingest). */ FIRST_STAGE_FILES_ONLY, /** @@ -1095,6 +1096,15 @@ final class IngestJobPipeline { this.checkForStageCompleted(); } + /** + * RJCTODO + * + * @param artifacts + */ + void addDataArtifacts(List artifacts) { + // RJCTODO + } + /** * Updates the display name shown on the current data source level ingest * progress bar for this job. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index fe581ca2d5..69afdbc85e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -302,10 +302,11 @@ final class IngestTasksScheduler { long jobId = ingestJobPipeline.getId(); return !(this.dataSourceIngestThreadQueue.hasTasksForJob(jobId) - || hasTasksForJob(this.rootFileTaskQueue, jobId) - || hasTasksForJob(this.pendingFileTaskQueue, jobId) - || hasTasksForJob(this.streamedTasksQueue, jobId) - || this.fileIngestThreadsQueue.hasTasksForJob(jobId)); + || hasTasksForJob(rootFileTaskQueue, jobId) + || hasTasksForJob(pendingFileTaskQueue, jobId) + || hasTasksForJob(streamedTasksQueue, jobId) + || fileIngestThreadsQueue.hasTasksForJob(jobId) + || dataArtifactIngestThreadQueue.hasTasksForJob(jobId)); } /** @@ -690,6 +691,7 @@ final class IngestTasksScheduler { this.fileIngestThreadsQueue.countQueuedTasksForJob(jobId), this.dataSourceIngestThreadQueue.countRunningTasksForJob(jobId) + this.fileIngestThreadsQueue.countRunningTasksForJob(jobId), countTasksForJob(this.streamedTasksQueue, jobId)); + // RJCTODO } /** From 27695de254d53a4d6e108ab5fc5ac4c3ba6fbbcc Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 1 Mar 2021 10:21:44 -0500 Subject: [PATCH 03/63] 7332 new ingest task pipeline/module type --- .../ingest/DataArtifactIngestTask.java | 17 +- .../autopsy/ingest/DataSourceIngestTask.java | 16 +- .../autopsy/ingest/FileIngestTask.java | 29 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 2 +- .../autopsy/ingest/IngestJobPipeline.java | 587 ++++++++++-------- .../autopsy/ingest/IngestJobSettings.java | 10 +- .../autopsy/ingest/IngestModuleTemplate.java | 10 +- .../sleuthkit/autopsy/ingest/IngestTask.java | 32 +- .../autopsy/ingest/IngestTasksScheduler.java | 2 +- 9 files changed, 370 insertions(+), 335 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java index e3fbf6a688..d03757b467 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java @@ -32,7 +32,7 @@ final class DataArtifactIngestTask extends IngestTask { * Constructs a data artifact ingest task that will be executed by an ingest * thread using a given ingest job pipeline. * - * @param ingestJobPipeline The ingest job pipeline to use to complete the + * @param ingestJobPipeline The ingest job pipeline to use to execute the * task. * @param artifact The data artifact to be processed. */ @@ -50,19 +50,4 @@ final class DataArtifactIngestTask extends IngestTask { return artifact; } - /** - * Executes this task by passing it to the given ingest job pipeline. - * - * @param ingestJobPipeline The ingest job pipeline. - * - * @throws InterruptedException This exception is thrown if the thread - * executing the task is interrupted while - * blocked. - */ - @Override - void execute(IngestJobPipeline ingestJobPipeline) throws InterruptedException { - // RJCTODO: Need overload for data artifacts - // ingestJobPipeline.process(this); - } - } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index dd7e217044..93f01def3d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -28,25 +28,11 @@ final class DataSourceIngestTask extends IngestTask { * Constructs a data source level ingest task that will be executed by an * ingest thread using a given ingest job pipeline. * - * @param ingestJobPipeline The ingest job pipeline to use to complete the + * @param ingestJobPipeline The ingest job pipeline to use to execute the * task. */ DataSourceIngestTask(IngestJobPipeline ingestJobPipeline) { super(ingestJobPipeline); } - /** - * Executes this task by passing it to the given ingest job pipeline. - * - * @param ingestJobPipeline The ingest job pipeline. - * - * @throws InterruptedException This exception is thrown if the thread - * executing the task is interrupted while - * blocked. - */ - @Override - void execute(IngestJobPipeline ingestJobPipeline) throws InterruptedException { - ingestJobPipeline.process(this); - } - } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 067523f7c1..6dfc0729d3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -30,13 +30,13 @@ import org.sleuthkit.datamodel.TskCoreException; final class FileIngestTask extends IngestTask { private final long fileId; - private AbstractFile file = null; + private AbstractFile file; /** * Constructs a file ingest task that will be executed by an ingest thread * using a given ingest job pipeline. * - * @param ingestJobPipeline The ingest job pipeline to use to complete the + * @param ingestJobPipeline The ingest job pipeline to use to execute the * task. * @param file The file to be processed. */ @@ -46,8 +46,19 @@ final class FileIngestTask extends IngestTask { fileId = file.getId(); } + /** + * Constructs a file ingest task that will be executed by an ingest thread + * using a given ingest job pipeline. This constructor supports streaming + * ingest by deferring the construction of the AbstractFile object for this + * task to conserve heap memory. + * + * @param ingestJobPipeline The ingest job pipeline to use to execute the + * task. + * @param fileId The object ID of the file to be processed. + */ FileIngestTask(IngestJobPipeline ingestJobPipeline, long fileId) { super(ingestJobPipeline); + file = null; this.fileId = fileId; } @@ -75,20 +86,6 @@ final class FileIngestTask extends IngestTask { return file; } - /** - * Executes this task by passing it to the given ingest job pipeline. - * - * @param ingestJobPipeline The ingest job pipeline. - * - * @throws InterruptedException This exception is thrown if the thread - * executing the task is interrupted while - * blocked. - */ - @Override - void execute(IngestJobPipeline ingestJobPipeline) throws InterruptedException { - ingestJobPipeline.process(this); - } - @Override public boolean equals(Object obj) { if (obj == null) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index decccd4e84..32a077ae59 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -210,7 +210,7 @@ public final class IngestJob { */ List errors = new ArrayList<>(); for (IngestJobPipeline ingestJobPipeline : this.ingestJobPipelines.values()) { - errors.addAll(ingestJobPipeline.start()); + errors.addAll(ingestJobPipeline.startUp()); if (errors.isEmpty() == false) { break; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 1a97b9e161..afcf1f2eb9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -52,56 +52,50 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.modules.interestingitems.FilesSet; import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer; +import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.DataArtifact; /** - * Encapsulates a data source and the ingest module pipelines used to process - * it. + * A set of ingest module pipelines for an ingest job. */ final class IngestJobPipeline { private static String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy"; - - private static final Logger logger = Logger.getLogger(IngestJobPipeline.class.getName()); - // to match something like: "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" private static final Pattern JYTHON_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); + private static final Logger logger = Logger.getLogger(IngestJobPipeline.class.getName()); - /** - * These fields define a data source ingest job: the parent ingest job, an - * ID, the user's ingest job settings, and the data source to be analyzed. - * Optionally, there is a set of files to be analyzed instead of analyzing - * all of the files in the data source. - */ - private final IngestJob parentJob; - private static final AtomicLong nextJobId = new AtomicLong(0L); - private final long id; + private final IngestJob job; + private static final AtomicLong nextPipelineId = new AtomicLong(0L); + private final long pipelineId; private final IngestJobSettings settings; - private Content dataSource = null; + private Content dataSource; private final IngestJob.Mode ingestMode; private final List files = new ArrayList<>(); + private final long createTime; - /** - * A data source ingest job runs in stages. + /* + * An ingest job pipeline runs in stages. */ private static enum Stages { - /** + /* * Setting up for processing. */ INITIALIZATION, - /** - * Running only file ingest modules (used only for streaming ingest). + /* + * Running only file and data artifact ingest modules (used only for + * streaming ingest). */ FIRST_STAGE_FILES_ONLY, - /** - * Running high priority data source level ingest modules and file level - * ingest modules. + /* + * Running high priority data source level ingest modules, file ingest + * modules and data artifact ingest modules. */ FIRST_STAGE_FILES_AND_DATASOURCE, /** * Running lower priority, usually long-running, data source level - * ingest modules. + * ingest modules and data artifact ingest modules. */ SECOND_STAGE, /** @@ -113,7 +107,7 @@ final class IngestJobPipeline { private final Object stageCompletionCheckLock = new Object(); /** - * A data source ingest job has separate data source level ingest module + * An ingest job pipeline has separate data source level ingest task * pipelines for the first and second processing stages. Longer running, * lower priority modules belong in the second stage pipeline, although this * cannot be enforced. Note that the pipelines for both stages are created @@ -126,25 +120,29 @@ final class IngestJobPipeline { private DataSourceIngestPipeline currentDataSourceIngestPipeline; /** - * A data source ingest job has a collection of identical file level ingest - * module pipelines, one for each file level ingest thread in the ingest - * manager. A blocking queue is used to dole out the pipelines to the - * threads and an ordinary list is used when the ingest job needs to access - * the pipelines to query their status. + * An ingest job pipeline has a collection of identical file ingest task + * pipelines, one for each file ingest thread in the ingest manager. The + * ingest threads take and return task pipelines using a blocking queue. A + * fixed list of all of the task pipelines is used to allow the job pipeline + * to cycle through the individual task pipelines to check their status and + * their progress on their current tasks. */ private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); private final List fileIngestPipelines = new ArrayList<>(); + /* + * An ingest job pipeline has a single data aritfact task pipeline. + */ + private DataArtifactIngestPipeline artifactIngestPipeline; + /** - * A data source ingest job supports cancellation of either the currently + * An ingest job pipeline supports cancellation of either the currently * running data source level ingest module or the entire ingest job. - * - * TODO: The currentDataSourceIngestModuleCancelled field and all of the - * code concerned with it is a hack to avoid an API change. The next time an - * API change is legal, a cancel() method needs to be added to the - * IngestModule interface and this field should be removed. The "ingest job - * is canceled" queries should also be removed from the IngestJobContext - * class. + * Cancellation works by setting flags that are checked by the ingest task + * pipelines every time they transition from from one module to another. + * Modules are also expected to check these flags and stop processing if + * they are set. This means that there can be a variable length delay + * between a cancellation request and its fulfillment. */ private volatile boolean currentDataSourceIngestModuleCancelled; private final List cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>(); @@ -152,28 +150,24 @@ final class IngestJobPipeline { private volatile IngestJob.CancellationReason cancellationReason = IngestJob.CancellationReason.NOT_CANCELLED; /** - * A data source ingest job uses the task scheduler singleton to create and - * queue the ingest tasks that make up the job. + * An ingest job pipeline interacts with the ingest task scheduler to + * determine whether or not there are ingest tasks still to be executed for + * an ingest job and to schedule additional tasks submitted by ingest + * modules. For example, a file carving module can add carved files to an + * ingest job and many modules will add data artifacts to an ingest job. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); /** - * A data source ingest job can run interactively using NetBeans progress - * handles. + * If running in a GUI, an ingest job pipeline report progress and allows a + * user to cancel a data source level ingest module or the entire ingest job + * using progress bars in the lower right hand corner of the main + * application window. This is one of a handful of things that currently + * couples the ingest code to the presentation layer. */ private final boolean doUI; - - /** - * A data source ingest job uses these fields to report data source level - * ingest progress. - */ private final Object dataSourceIngestProgressLock = new Object(); private ProgressHandle dataSourceIngestProgress; - - /** - * A data source ingest job uses these fields to report file level ingest - * progress. - */ private final Object fileIngestProgressLock = new Object(); private final List filesInProgress = new ArrayList<>(); private long estimatedFilesToProcess; @@ -185,87 +179,77 @@ final class IngestJobPipeline { private volatile IngestJobInfo ingestJob; /** - * A data source ingest job uses this field to report its creation time. - */ - private final long createTime; - - /** - * Constructs an object that encapsulates a data source and the ingest - * module pipelines used to analyze it. + * Constructs a set of ingest module pipelines for an ingest job. * - * @param parentJob The ingest job of which this data source ingest job is - * a part. - * @param dataSource The data source to be ingested. - * @param settings The settings for the ingest job. + * @param job The ingest job. + * @param dataSource The data source for the ingest job. + * @param settings The ingest settings for the ingest job. */ - IngestJobPipeline(IngestJob parentJob, Content dataSource, IngestJobSettings settings) { - this(parentJob, dataSource, Collections.emptyList(), settings); + IngestJobPipeline(IngestJob job, Content dataSource, IngestJobSettings settings) { + this(job, dataSource, Collections.emptyList(), settings); } /** - * Constructs an object that encapsulates a data source and the ingest - * module pipelines used to analyze it. Either all of the files in the data - * source or a given subset of the files will be analyzed. + * Constructs a set of ingest module pipelines for an ingest job that + * process a subset of the files in the job's data source. * - * @param parentJob The ingest job of which this data source ingest job is - * a part. - * @param dataSource The data source to be ingested. - * @param files A subset of the files for the data source. - * @param settings The settings for the ingest job. + * @param job The ingest job. + * @param dataSource The data source for the ingest job. + * @param files The subset of the files for the data source. If the + * list is empty, all of the files in the data source are + * processed. + * @param settings The ingest settings for the ingest job. */ - IngestJobPipeline(IngestJob parentJob, Content dataSource, List files, IngestJobSettings settings) { - this.parentJob = parentJob; - this.id = IngestJobPipeline.nextJobId.getAndIncrement(); + IngestJobPipeline(IngestJob job, Content dataSource, List files, IngestJobSettings settings) { + this.job = job; this.dataSource = dataSource; - this.files.addAll(files); - this.ingestMode = parentJob.getIngestMode(); this.settings = settings; - this.doUI = RuntimeProperties.runningWithGUI(); - this.createTime = new Date().getTime(); - this.stage = Stages.INITIALIZATION; - this.createIngestPipelines(); + this.files.addAll(files); + pipelineId = IngestJobPipeline.nextPipelineId.getAndIncrement(); + ingestMode = job.getIngestMode(); + doUI = RuntimeProperties.runningWithGUI(); + createTime = new Date().getTime(); + stage = Stages.INITIALIZATION; + currentFileIngestModule = ""; + currentFileIngestTask = ""; + createIngestTaskPipelines(); } /** - * Adds ingest modules to a list with autopsy modules first and third party - * modules next. + * Adds ingest module templates to an output list with core Autopsy modules + * first and third party modules next. * - * @param dest The destination for the modules to be added. - * @param src A map of fully qualified class name mapped to the - * IngestModuleTemplate. - * @param jythonSrc A map of fully qualified class name mapped to the - * IngestModuleTemplate for jython modules. + * @param orderedModules The list to populate. + * @param javaModules The input ingest module templates for modules + * implemented using Java. + * @param jythonModules The input ingest module templates for modules + * implemented using Jython. */ - private static void addOrdered(final List dest, - final Map src, final Map jythonSrc) { - + private static void addOrdered(final List orderedModules, final Map javaModules, final Map jythonModules) { final List autopsyModules = new ArrayList<>(); final List thirdPartyModules = new ArrayList<>(); - - Stream.concat(src.entrySet().stream(), jythonSrc.entrySet().stream()).forEach((templateEntry) -> { + Stream.concat(javaModules.entrySet().stream(), jythonModules.entrySet().stream()).forEach((templateEntry) -> { if (templateEntry.getKey().startsWith(AUTOPSY_MODULE_PREFIX)) { autopsyModules.add(templateEntry.getValue()); } else { thirdPartyModules.add(templateEntry.getValue()); } }); - - dest.addAll(autopsyModules); - dest.addAll(thirdPartyModules); + orderedModules.addAll(autopsyModules); + orderedModules.addAll(thirdPartyModules); } /** - * Takes a classname like + * Takes a Jython proxy class name like * "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" - * and provides "GPX_Parser_Module.GPXParserFileIngestModuleFactory" or null - * if not in jython package. + * and returns "GPX_Parser_Module.GPXParserFileIngestModuleFactory." * - * @param canonicalName The canonical name. + * @param className The class name. * * @return The jython name or null if not in jython package. */ - private static String getJythonName(String canonicalName) { - Matcher m = JYTHON_REGEX.matcher(canonicalName); + private static String getModuleNameFromJythonClassName(String className) { + Matcher m = JYTHON_REGEX.matcher(className); if (m.find()) { return String.format("%s.%s", m.group(1), m.group(2)); } else { @@ -274,19 +258,18 @@ final class IngestJobPipeline { } /** - * Adds a template to the appropriate map. If the class is a jython class, - * then it is added to the jython map. Otherwise, it is added to the - * mapping. + * Adds an ingest module template to one of two mappings of ingest module + * factory class names to module templates. One mapping is for ingest + * modules imnplemented using Java and the other is for ingest modules + * implemented using Jython. * - * @param mapping Mapping for non-jython objects. - * @param jythonMapping Mapping for jython objects. - * @param template The template to add. + * @param mapping Mapping for Java ingest module templates. + * @param jythonMapping Mapping for Jython ingest module templates. + * @param template The ingest module template. */ - private static void addModule(Map mapping, - Map jythonMapping, IngestModuleTemplate template) { - + private static void addModule(Map mapping, Map jythonMapping, IngestModuleTemplate template) { String className = template.getModuleFactory().getClass().getCanonicalName(); - String jythonName = getJythonName(className); + String jythonName = getModuleNameFromJythonClassName(className); if (jythonName != null) { jythonMapping.put(jythonName, template); } else { @@ -295,61 +278,71 @@ final class IngestJobPipeline { } /** - * Creates the file and data source ingest pipelines. + * Creates the ingest task pipelines for this ingest job pipeline. */ - private void createIngestPipelines() { - List ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates(); + private void createIngestTaskPipelines() { + /* + * Get the complete set of ingest module templates from the ingest job + * settings. An ingest module template combines an ingest module factory + * with job level ingest module settings to support the creation of any + * number of fully configured instances of a given ingest module. + */ + List enabledIngestModuleTemplates = this.settings.getEnabledIngestModuleTemplates(); /** - * Make mappings of ingest module factory class names to templates. + * Make one mapping of ingest module factory class names to ingest + * module templates for each type of ingest task pipeline to be created. + * These mappings are used to go from an ingest module factory class + * name in the pipeline configuration file to the corresponding ingest + * module template. */ - Map dataSourceModuleTemplates = new LinkedHashMap<>(); - Map fileModuleTemplates = new LinkedHashMap<>(); - - // mappings for jython modules. These mappings are only used to determine modules in the pipelineconfig.xml. - Map jythonDataSourceModuleTemplates = new LinkedHashMap<>(); - Map jythonFileModuleTemplates = new LinkedHashMap<>(); - - for (IngestModuleTemplate template : ingestModuleTemplates) { + Map unorderedDataSourceModuleTemplates = new LinkedHashMap<>(); + Map unorderedJythonDataSourceModuleTemplates = new LinkedHashMap<>(); + Map unorderedFileModuleTemplates = new LinkedHashMap<>(); + Map unorderedJythonFileModuleTemplates = new LinkedHashMap<>(); + Map unorderedArtifactModuleTemplates = new LinkedHashMap<>(); + Map unorderedJythonArtifactModuleTemplates = new LinkedHashMap<>(); + for (IngestModuleTemplate template : enabledIngestModuleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - addModule(dataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); + addModule(unorderedDataSourceModuleTemplates, unorderedJythonDataSourceModuleTemplates, template); + continue; } if (template.isFileIngestModuleTemplate()) { - addModule(fileModuleTemplates, jythonFileModuleTemplates, template); + addModule(unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates, template); + continue; + } + if (template.isDataArtifactIngestModuleTemplate()) { + addModule(unorderedArtifactModuleTemplates, unorderedJythonArtifactModuleTemplates, template); } } /** - * Use the mappings and the ingest pipelines configuration to create - * ordered lists of ingest module templates for each ingest pipeline. + * Use the mappings and the entries read from the ingest pipelines + * configuration file to create ordered lists of ingest module template + * for each ingest task pipeline. */ IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates( - dataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - - List fileIngestModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates( - fileModuleTemplates, jythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - - List secondStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates( - dataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + List firstStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, unorderedJythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); + List fileIngestModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); + List secondStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); /** - * Add any module templates that were not specified in the pipelines - * configuration to an appropriate pipeline - either the first stage - * data source ingest pipeline or the file ingest pipeline. + * Add any module templates that were not specified in the pipeline + * configuration file to the appropriate list. Note that no data + * artifact ingest modules are currently specified in the file and that + * core Autopsy modules are added first and third party modules are + * added last. */ - addOrdered(firstStageDataSourceModuleTemplates, dataSourceModuleTemplates, jythonDataSourceModuleTemplates); - addOrdered(fileIngestModuleTemplates, fileModuleTemplates, jythonFileModuleTemplates); + addOrdered(firstStageDataSourceModuleTemplates, unorderedDataSourceModuleTemplates, unorderedJythonDataSourceModuleTemplates); + addOrdered(fileIngestModuleTemplates, unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates); + addOrdered(fileIngestModuleTemplates, unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates); + List artifactModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); /** - * Construct the data source ingest pipelines. - */ - this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); - this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); - - /** - * Construct the file ingest pipelines, one per file ingest thread. + * Construct the ingest task pipelines from the ordered lists. */ + firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); + secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); try { int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); for (int i = 0; i < numberOfFileIngestThreads; ++i) { @@ -365,16 +358,33 @@ final class IngestJobPipeline { */ Thread.currentThread().interrupt(); } + artifactIngestPipeline = new DataArtifactIngestPipeline(this, artifactModuleTemplates); + + /* + * Add the ingest module templates to the "master list" and record the + * final composition of the ingest task pipelines in the case database. + */ try { SleuthkitCase skCase = Case.getCurrentCaseThrows().getSleuthkitCase(); - this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); - this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase); - this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); + addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); + addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase); + addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); } catch (TskCoreException | NoCurrentCaseException ex) { logErrorMessage(Level.WARNING, "Failed to add ingest modules listing to case database", ex); } } + /** + * Adds a list of ingest module templates for an ingest task pipeline to the + * "master list" and records the final composition of the pipeline in the + * case database. + * + * @param templates A list of ingest module templates. + * @param type The type of the ingest module templates. + * @param skCase The case database. + * + * @throws TskCoreException + */ private void addIngestModules(List templates, IngestModuleType type, SleuthkitCase skCase) throws TskCoreException { for (IngestModuleTemplate module : templates) { ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber())); @@ -385,28 +395,28 @@ final class IngestJobPipeline { * Uses an input collection of ingest module templates and a pipeline * configuration, i.e., an ordered list of ingest module factory class * names, to create an ordered output list of ingest module templates for an - * ingest pipeline. The ingest module templates are removed from the input - * collection as they are added to the output collection. + * ingest task pipeline. The ingest module templates are removed from the + * input collection as they are added to the output collection. * - * @param ingestModuleTemplates A mapping of ingest module factory - * class names to ingest module - * templates. - * @param jythonIngestModuleTemplates A mapping of jython processed class - * names to jython ingest module + * @param javaIngestModuleTemplates A mapping of Java ingest module + * factory class names to ingest module * templates. + * @param jythonIngestModuleTemplates A mapping of Jython ingest module + * factory proxy class names to ingest + * module templates. * @param pipelineConfig An ordered list of ingest module * factory class names representing an - * ingest pipeline. + * ingest pipeline, read form the + * pipeline configuration file. * * @return An ordered list of ingest module templates, i.e., an * uninstantiated pipeline. */ - private static List getConfiguredIngestModuleTemplates( - Map ingestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { + private static List getConfiguredIngestModuleTemplates(Map javaIngestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { List templates = new ArrayList<>(); for (String moduleClassName : pipelineConfig) { - if (ingestModuleTemplates != null && ingestModuleTemplates.containsKey(moduleClassName)) { - templates.add(ingestModuleTemplates.remove(moduleClassName)); + if (javaIngestModuleTemplates != null && javaIngestModuleTemplates.containsKey(moduleClassName)) { + templates.add(javaIngestModuleTemplates.remove(moduleClassName)); } else if (jythonIngestModuleTemplates != null && jythonIngestModuleTemplates.containsKey(moduleClassName)) { templates.add(jythonIngestModuleTemplates.remove(moduleClassName)); } @@ -415,25 +425,25 @@ final class IngestJobPipeline { } /** - * Gets the identifier of this job. + * Gets the ID of this ingest job pipeline. * - * @return The job identifier. + * @return The job ID. */ long getId() { - return this.id; + return this.pipelineId; } /** - * Get the ingest execution context identifier. + * Gets the ingest execution context name. * - * @return The context string. + * @return The context name. */ String getExecutionContext() { return this.settings.getExecutionContext(); } /** - * Gets the data source to be ingested by this job. + * Getss the data source to be ingested by this ingest job pipeline. * * @return A Content object representing the data source. */ @@ -442,8 +452,8 @@ final class IngestJobPipeline { } /** - * Queries whether or not unallocated space should be processed as part of - * this job. + * Queries whether or not unallocated space should be processed by this + * ingest job pipeline. * * @return True or false. */ @@ -452,28 +462,31 @@ final class IngestJobPipeline { } /** - * Gets the selected file ingest filter from settings. + * Gets the file ingest filter for this ingest job pipeline from the ingest + * job settings. * - * @return True or false. + * @return The filter. */ FilesSet getFileIngestFilter() { return this.settings.getFileFilter(); } /** - * Checks to see if this job has at least one ingest pipeline. + * Checks to see if this ingest job pipeline has at least one ingest task + * pipeline. * * @return True or false. */ boolean hasIngestPipeline() { - return this.hasFirstStageDataSourceIngestPipeline() - || this.hasFileIngestPipeline() - || this.hasSecondStageDataSourceIngestPipeline(); + return hasFirstStageDataSourceIngestPipeline() + || hasFileIngestPipeline() + || hasSecondStageDataSourceIngestPipeline() + || hasDataArtifactPipeline(); } /** - * Checks to see if this job has a first stage data source level ingest - * pipeline. + * Checks to see if this ingest job pipeline has a first stage data source + * level ingest pipeline. * * @return True or false. */ @@ -482,8 +495,8 @@ final class IngestJobPipeline { } /** - * Checks to see if this job has a second stage data source level ingest - * pipeline. + * Checks to see if this ingest job pipeline has a second stage data source + * level ingest task pipeline. * * @return True or false. */ @@ -492,7 +505,8 @@ final class IngestJobPipeline { } /** - * Checks to see if this job has a file level ingest pipeline. + * Checks to see if this ingest job pipeline has a file level ingest task + * pipeline. * * @return True or false. */ @@ -504,92 +518,121 @@ final class IngestJobPipeline { } /** - * Starts up the ingest pipelines for this job. + * Checks to see if this ingest job pipeline has a data artifact ingest task + * pipeline. + * + * @return True or false. + */ + private boolean hasDataArtifactPipeline() { + return (artifactIngestPipeline.isEmpty() == false); + } + + /** + * Starts up this ingest job pipeline. * * @return A collection of ingest module startup errors, empty on success. */ - List start() { + List startUp() { if (dataSource == null) { - // TODO - Remove once data source is always present during initialization throw new IllegalStateException("Ingest started before setting data source"); } - List errors = startUpIngestPipelines(); + List errors = startUpIngestTaskPipelines(); if (errors.isEmpty()) { try { - this.ingestJob = Case.getCurrentCaseThrows().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); + ingestJob = Case.getCurrentCaseThrows().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); } catch (TskCoreException | NoCurrentCaseException ex) { logErrorMessage(Level.WARNING, "Failed to add ingest job info to case database", ex); //NON-NLS } - if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { + startArtifactProcessing(); + if (hasFirstStageDataSourceIngestPipeline() || hasFileIngestPipeline()) { if (ingestMode == IngestJob.Mode.BATCH) { - logInfoMessage("Starting first stage analysis"); //NON-NLS - this.startFirstStage(); + logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS + startFirstStage(); } else { - logInfoMessage("Preparing for first stage analysis"); //NON-NLS - this.startFileIngestStreaming(); + logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS + startFileIngestStreaming(); } - } else if (this.hasSecondStageDataSourceIngestPipeline()) { + } else if (hasSecondStageDataSourceIngestPipeline()) { logInfoMessage("Starting second stage analysis"); //NON-NLS - this.startSecondStage(); + startSecondStage(); } } return errors; } /** - * Starts up each of the ingest pipelines for this job to collect any file - * and data source level ingest modules errors that might occur. + * Starts up each of the ingest task pipelines in this ingest job pipeline. + * All of the pipelines are started so that any and all start up errors can + * be returned to the caller. It is important to capture all of the errors + * since the ingest job will be automatically cancelled and the errors will + * be reported to the user so the issues can be addressed or the modules + * that can't start up can be disabled before the ingest job is attempted + * again. * - * @return A collection of ingest module startup errors, empty on success. + * @return A list of ingest module startup errors, empty on success. */ - private List startUpIngestPipelines() { + private List startUpIngestTaskPipelines() { List errors = new ArrayList<>(); - - /* - * Start the data-source-level ingest module pipelines. - */ - errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); - errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); - - /* - * If the data-source-level ingest pipelines were successfully started, - * start the file-level ingest pipelines (one per pipeline file ingest - * thread). - */ - if (errors.isEmpty()) { - for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) { - errors.addAll(pipeline.startUp()); - if (!errors.isEmpty()) { - /* - * If there are start up errors, the ingest job will not - * proceed, so shut down any file ingest pipelines that did - * start up. - */ - while (!this.fileIngestPipelinesQueue.isEmpty()) { - FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll(); - if (startedPipeline.isRunning()) { - List shutDownErrors = startedPipeline.shutDown(); - if (!shutDownErrors.isEmpty()) { - /* - * The start up errors will ultimately be - * reported to the user for possible remedy, but - * the shut down errors are logged here. - */ - logIngestModuleErrors(shutDownErrors); - } - } - } - break; - } + startUpIngestTaskPipeline(firstStageDataSourceIngestPipeline); + startUpIngestTaskPipeline(secondStageDataSourceIngestPipeline); + for (FileIngestPipeline pipeline : fileIngestPipelines) { + List filePipelineErrors = startUpIngestTaskPipeline(pipeline); + if (!filePipelineErrors.isEmpty()) { + /* + * If one pipeline copy can't start up, assume that none of them + * will be able to start up for the same reasons. + */ + errors.addAll(filePipelineErrors); + break; } } - + errors.addAll(this.artifactIngestPipeline.startUp()); return errors; } /** - * Starts the first stage of this job. + * Starts up an ingest task pipeline. If there are any start up errors, the + * piepline is imediately shut down. + * + * @param pipeline The ingest task pipeline to start up. + * + * @return A list of ingest module startup errors, empty on success. + */ + private List startUpIngestTaskPipeline(IngestTaskPipeline pipeline) { + List startUpErrors = pipeline.startUp(); + if (!startUpErrors.isEmpty()) { + List shutDownErrors = pipeline.shutDown(); + if (!shutDownErrors.isEmpty()) { + /* + * The start up errors will ultimately be reported to the user + * for possible remedy, but the shut down errors are logged + * here. + */ + logIngestModuleErrors(shutDownErrors); + } + } + return startUpErrors; + } + + /** + * Schedules data artifact tasks for all of the existing data artifacts for + * the ingest job's data source, if any. Note that there is a possiblity, + * just as in with the other item types, that a user may process artifacts + * more than once if the same ingest job is run more than once. + */ + private void startArtifactProcessing() { + Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); + try { + List artifacts = blackboard.getDataArtifacts(dataSource); + taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); + } catch (TskCoreException ex) { + // RJCTODO + } + } + + /** + * Starts the first stage of the ingest job. */ private void startFirstStage() { this.stage = IngestJobPipeline.Stages.FIRST_STAGE_FILES_AND_DATASOURCE; @@ -880,7 +923,7 @@ final class IngestJobPipeline { } } - /** + /* * Start the second stage, if appropriate. */ if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { @@ -927,16 +970,37 @@ final class IngestJobPipeline { logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } } - this.parentJob.ingestJobPipelineFinished(this); + this.job.ingestJobPipelineFinished(this); } /** - * Passes the data source for this job through the currently active data + * Executes an ingest task for an ingest job by passing the item associated + * with the task through the appropriate pipeline of ingest modules. + * + * @param task The ingest task. + */ + void execute(IngestTask task) { + /* + * The following "switch on actual type" eliminates code duplication in + * the IngestTask hierarchy. Future work may or may not be able to + * eliminate the switch. + */ + if (task instanceof DataSourceIngestTask) { + executeDataSourceIngestTask((DataSourceIngestTask) task); + } else if (task instanceof FileIngestTask) { + executeFileIngestTask((FileIngestTask) task); + } else if (task instanceof DataArtifactIngestTask) { + executeDataArtifactIngestTask((DataArtifactIngestTask) task); + } + } + + /** + * Passes a data source for this job through the currently active data * source level ingest pipeline. * * @param task A data source ingest task wrapping the data source. */ - void process(DataSourceIngestTask task) { + private void executeDataSourceIngestTask(DataSourceIngestTask task) { try { synchronized (this.dataSourceIngestPipelineLock) { if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { @@ -968,17 +1032,12 @@ final class IngestJobPipeline { } /** - * Passes a file from the data source for this job through the file level - * ingest pipeline. + * Passes a file from the data source for this job through the file ingest + * pipeline. * - * @param task A file ingest task. - * - * @throws InterruptedException if the thread executing this code is - * interrupted while blocked on taking from or - * putting to the file ingest pipelines - * collection. + * @param task A file ingest task wrapping the file. */ - void process(FileIngestTask task) throws InterruptedException { + private void executeFileIngestTask(FileIngestTask task) { try { if (!this.isCancelled()) { FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take(); @@ -1037,12 +1096,24 @@ final class IngestJobPipeline { } this.fileIngestPipelinesQueue.put(pipeline); } + } catch (InterruptedException ex) { + // RJCTODO } finally { IngestJobPipeline.taskScheduler.notifyTaskCompleted(task); this.checkForStageCompleted(); } } + /** + * Passes a data artifact from the data source for this job through the data + * artifact ingest pipeline. + * + * @param task A data artifact ingest task wrapping the file. + */ + private void executeDataArtifactIngestTask(DataArtifactIngestTask task) { + // RJCTODO + } + /** * Add a list of files (by object ID) to the ingest queue. Must call start() * prior to adding files. @@ -1297,8 +1368,7 @@ final class IngestJobPipeline { * @param taskName Name of file the module is running on. */ void setCurrentFileIngestModule(String moduleName, String taskName) { - this.currentFileIngestModule = moduleName; - this.currentFileIngestTask = taskName; + currentFileIngestTask = taskName; } /** @@ -1327,7 +1397,7 @@ final class IngestJobPipeline { * @param message The message. */ private void logInfoMessage(String message) { - logger.log(Level.INFO, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), id, ingestJob.getIngestJobId())); //NON-NLS + logger.log(Level.INFO, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJob.getIngestJobId())); //NON-NLS } /** @@ -1339,7 +1409,7 @@ final class IngestJobPipeline { * @param throwable The throwable associated with the error. */ private void logErrorMessage(Level level, String message, Throwable throwable) { - logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), id, ingestJob.getIngestJobId()), throwable); //NON-NLS + logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJob.getIngestJobId()), throwable); //NON-NLS } /** @@ -1350,7 +1420,7 @@ final class IngestJobPipeline { * @param message The message. */ private void logErrorMessage(Level level, String message) { - logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id %d)", message, this.dataSource.getName(), this.dataSource.getId(), id, ingestJob.getIngestJobId())); //NON-NLS + logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJob.getIngestJobId())); //NON-NLS } /** @@ -1411,13 +1481,14 @@ final class IngestJobPipeline { estimatedFilesToProcessCount = this.estimatedFilesToProcess; snapShotTime = new Date().getTime(); } - tasksSnapshot = IngestJobPipeline.taskScheduler.getTasksSnapshotForJob(id); + tasksSnapshot = IngestJobPipeline.taskScheduler.getTasksSnapshotForJob(pipelineId); } - return new Snapshot(this.dataSource.getName(), id, createTime, + return new Snapshot(this.dataSource.getName(), pipelineId, createTime, getCurrentDataSourceIngestModule(), fileIngestRunning, fileIngestStartTime, cancelled, cancellationReason, cancelledDataSourceIngestModules, processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot); } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java index 918c30760f..7de83ded4c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2018 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -318,7 +318,7 @@ public final class IngestJobSettings { // Add modules that are going to be used for this ingest depending on type. for (IngestModuleFactory moduleFactory : allModuleFactories) { - if (this.ingestType.equals(IngestType.ALL_MODULES)) { + if (moduleFactory.isDataArtifactIngestModuleFactory() || ingestType.equals(IngestType.ALL_MODULES)) { moduleFactories.add(moduleFactory); } else if (this.ingestType.equals(IngestType.DATA_SOURCE_ONLY) && moduleFactory.isDataSourceIngestModuleFactory()) { moduleFactories.add(moduleFactory); @@ -330,7 +330,7 @@ public final class IngestJobSettings { for (IngestModuleFactory moduleFactory : moduleFactories) { loadedModuleNames.add(moduleFactory.getModuleDisplayName()); } - + /** * Hard coding Plaso to be disabled by default. loadedModuleNames is * passed below as the default list of enabled modules so briefly remove @@ -361,7 +361,7 @@ public final class IngestJobSettings { if (plasoLoaded) { loadedModuleNames.add(plasoModuleName); } - + /** * Check for missing modules and create warnings if any are found. */ @@ -549,7 +549,7 @@ public final class IngestJobSettings { * @return The file path. */ private String getModuleSettingsFilePath(IngestModuleFactory factory) { - String fileName = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + IngestJobSettings.MODULE_SETTINGS_FILE_EXT; + String fileName = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + IngestJobSettings.MODULE_SETTINGS_FILE_EXT; Path path = Paths.get(this.moduleSettingsFolderPath, fileName); return path.toAbsolutePath().toString(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java index 83a290c78a..26285f6439 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2018 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -85,6 +85,14 @@ public final class IngestModuleTemplate { return moduleFactory.createFileIngestModule(settings); } + public boolean isDataArtifactIngestModuleTemplate() { + return moduleFactory.isDataArtifactIngestModuleFactory(); + } + + public DataArtifactIngestModule createDataArtifactIngestModule() { + return moduleFactory.createDataArtifactIngestModule(settings); + } + public void setEnabled(boolean enabled) { this.enabled = enabled; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index 261edeb24f..23a323690a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -22,23 +22,23 @@ import org.sleuthkit.datamodel.Content; /** * An ingest task that will be executed by an ingest thread using a given ingest - * job pipeline. The task is one unit of analysis work for an ingest job for a - * data source. Some examples of concrete types of ingest tasks include the - * analysis of files and the analysis of data artifacts. + * job pipeline. Three examples of concrete types of ingest tasks are tasks to + * analyze a data source, tasks to analyze the files in a data source, and tasks + * that analyze data artifacts. */ abstract class IngestTask { - + private final static long NOT_SET = Long.MIN_VALUE; private final IngestJobPipeline ingestJobPipeline; private long threadId; /** * Constructs an ingest task that will be executed by an ingest thread using - * a given ingest job pipeline. The task is one unit of analysis work for an - * ingest job for a data source. Some examples of concrete types of ingest - * tasks include the analysis of files and the analysis of data artifacts. + * a given ingest job pipeline. Three examples of concrete types of ingest + * tasks are tasks to analyze a data source, tasks to analyze the files in a + * data source, and tasks that analyze data artifacts. * - * @param ingestJobPipeline The ingest job pipeline to use to complete the + * @param ingestJobPipeline The ingest job pipeline to use to execute the * task. */ IngestTask(IngestJobPipeline ingestJobPipeline) { @@ -85,19 +85,7 @@ abstract class IngestTask { */ void execute(long threadId) throws InterruptedException { // RJCTODO: Why does htis block? this.threadId = threadId; - execute(ingestJobPipeline); + ingestJobPipeline.execute(this); } - - /** - * Executes this task using the ingest job pipeline specified when the task - * was created. - * - * @param ingestJobPipeline The ingest job pipeline. - * - * @throws InterruptedException This exception is thrown if the thread - * executing the task is interrupted while - * blocked. - */ - abstract void execute(IngestJobPipeline IngestJobPipeline) throws InterruptedException; - + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 69afdbc85e..7eedcf12aa 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -622,7 +622,7 @@ final class IngestTasksScheduler { return false; } } - + /** * Checks whether or not a collection of ingest tasks includes a task for a * given data source ingest job. From d9eddcef7d283a43835a3976502e86634424db02 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 8 Mar 2021 15:35:07 -0500 Subject: [PATCH 04/63] 7332 Artifacts ingest pipeline, interim commit --- .../CentralRepoDataArtifactIngestModule.java | 51 + .../CentralRepoIngestModuleFactory.java | 67 +- .../ingest/DataArtifactIngestPipeline.java | 6 +- .../ingest/DataArtifactIngestTask.java | 2 +- .../DataSourceIngestModuleProgress.java | 4 +- .../ingest/DataSourceIngestPipeline.java | 4 +- .../autopsy/ingest/DataSourceIngestTask.java | 2 +- .../autopsy/ingest/FileIngestPipeline.java | 6 +- .../autopsy/ingest/FileIngestTask.java | 8 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 81 +- .../autopsy/ingest/IngestJobContext.java | 4 +- ...stJobPipeline.java => IngestPipeline.java} | 947 ++++++++++-------- .../sleuthkit/autopsy/ingest/IngestTask.java | 6 +- .../autopsy/ingest/IngestTaskPipeline.java | 6 +- .../autopsy/ingest/IngestTasksScheduler.java | 51 +- 15 files changed, 747 insertions(+), 498 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java rename Core/src/org/sleuthkit/autopsy/ingest/{IngestJobPipeline.java => IngestPipeline.java} (61%) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java new file mode 100755 index 0000000000..278c6ba0e4 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java @@ -0,0 +1,51 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import java.util.concurrent.atomic.AtomicLong; +import org.sleuthkit.autopsy.ingest.DataArtifactIngestModule; +import org.sleuthkit.autopsy.ingest.IngestMessage; +import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.datamodel.DataArtifact; + +/** + * A placeholder data artifact ingest module that counts the number of data + * artifacts it processes and posts the final count to the ingest inbox. + */ +public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestModule { + + private final AtomicLong artifactCounter = new AtomicLong(); + + @Override + public ProcessResult process(DataArtifact artifact) { + artifactCounter.incrementAndGet(); + return ProcessResult.OK; + } + + @Override + public void shutDown() { + IngestServices.getInstance().postMessage(IngestMessage.createMessage( + IngestMessage.MessageType.INFO, + CentralRepoIngestModuleFactory.getModuleName(), + "Data Artifacts Processed", //NON-NLS + String.format("Count = %d", artifactCounter.get()))); //NON-NLS + + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java index 1c34f1ffad..b6ab3ae52d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2018 Basis Technology Corp. + * Copyright 2015-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,19 +26,20 @@ import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.DataArtifactIngestModule; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; /** - * Factory for Central Repository ingest modules + * An ingest module factory for Central Repository ingest modules. */ @ServiceProvider(service = org.sleuthkit.autopsy.ingest.IngestModuleFactory.class) @NbBundle.Messages({"CentralRepoIngestModuleFactory.ingestmodule.name=Central Repository", - "CentralRepoIngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation"}) + "CentralRepoIngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation"}) public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { /** - * Get the name of the module. + * Gets the name of the module. * * @return The module name. */ @@ -61,26 +62,6 @@ public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { return Version.getVersion(); } - @Override - public boolean isFileIngestModuleFactory() { - return true; - } - - @Override - public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { - if (settings instanceof IngestSettings) { - return new CentralRepoIngestModule((IngestSettings) settings); - } - /* - * Compatibility check for older versions. - */ - if (settings instanceof NoIngestModuleIngestJobSettings) { - return new CentralRepoIngestModule(new IngestSettings()); - } - - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); - } - @Override public boolean hasGlobalSettingsPanel() { return true; @@ -92,7 +73,7 @@ public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { globalOptionsPanel.load(); return globalOptionsPanel; } - + @Override public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { return new IngestSettings(); @@ -109,13 +90,43 @@ public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { return new IngestSettingsPanel((IngestSettings) settings); } /* - * Compatibility check for older versions. + * Earlier versions of the modules had no ingest job settings. Create a + * panel with the default settings. */ if (settings instanceof NoIngestModuleIngestJobSettings) { - return new IngestSettingsPanel(new IngestSettings()); + return new IngestSettingsPanel((IngestSettings) getDefaultIngestJobSettings()); } - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); } + @Override + public boolean isFileIngestModuleFactory() { + return true; + } + + @Override + public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { + if (settings instanceof IngestSettings) { + return new CentralRepoIngestModule((IngestSettings) settings); + } + /* + * Earlier versions of the modules had no ingest job settings. Create a + * module with the default settings. + */ + if (settings instanceof NoIngestModuleIngestJobSettings) { + return new CentralRepoIngestModule((IngestSettings) getDefaultIngestJobSettings()); + } + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); + } + + @Override + public boolean isDataArtifactIngestModuleFactory() { + return true; + } + + @Override + public DataArtifactIngestModule createDataArtifactIngestModule(IngestModuleIngestJobSettings settings) { + return new CentralRepoDataArtifactIngestModule(); + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java index cdd5d7937e..35382899c8 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java @@ -27,7 +27,7 @@ import java.util.Optional; */ final class DataArtifactIngestPipeline extends IngestTaskPipeline { - private final IngestJobPipeline ingestJobPipeline; + private final IngestPipeline ingestJobPipeline; /** * Constructs a pipeline of data artifact ingest modules for performing data @@ -37,7 +37,7 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline moduleTemplates) { + DataArtifactIngestPipeline(IngestPipeline ingestJobPipeline, List moduleTemplates) { super(ingestJobPipeline, moduleTemplates); this.ingestJobPipeline = ingestJobPipeline; } @@ -88,7 +88,7 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline moduleTemplates) { + DataSourceIngestPipeline(IngestPipeline ingestJobPipeline, List moduleTemplates) { super(ingestJobPipeline, moduleTemplates); } @@ -83,7 +83,7 @@ final class DataSourceIngestPipeline extends IngestTaskPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJobPipeline ingestJobPipeline; + private final IngestPipeline ingestJobPipeline; /** * Constructs a pipeline of file ingest modules for performing file ingest @@ -40,7 +40,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { * @param moduleTemplates The ingest module templates that define this * pipeline. */ - FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + FileIngestPipeline(IngestPipeline ingestJobPipeline, List moduleTemplates) { super(ingestJobPipeline, moduleTemplates); this.ingestJobPipeline = ingestJobPipeline; } @@ -140,7 +140,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { * org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException */ @Override - void performTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException { + void performTask(IngestPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException { AbstractFile file = null; try { file = task.getFile(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 6dfc0729d3..1b17c27f98 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -40,7 +40,7 @@ final class FileIngestTask extends IngestTask { * task. * @param file The file to be processed. */ - FileIngestTask(IngestJobPipeline ingestJobPipeline, AbstractFile file) { + FileIngestTask(IngestPipeline ingestJobPipeline, AbstractFile file) { super(ingestJobPipeline); this.file = file; fileId = file.getId(); @@ -56,7 +56,7 @@ final class FileIngestTask extends IngestTask { * task. * @param fileId The object ID of the file to be processed. */ - FileIngestTask(IngestJobPipeline ingestJobPipeline, long fileId) { + FileIngestTask(IngestPipeline ingestJobPipeline, long fileId) { super(ingestJobPipeline); file = null; this.fileId = fileId; @@ -95,8 +95,8 @@ final class FileIngestTask extends IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - IngestJobPipeline thisPipeline = getIngestJobPipeline(); - IngestJobPipeline otherPipeline = other.getIngestJobPipeline(); + IngestPipeline thisPipeline = getIngestJobPipeline(); + IngestPipeline otherPipeline = other.getIngestJobPipeline(); if (thisPipeline != otherPipeline && (thisPipeline == null || !thisPipeline.equals(otherPipeline))) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 32a077ae59..451a9b06cc 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2018 Basis Technology Corp. + * Copyright 2014-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -63,7 +63,7 @@ public final class IngestJob { return displayName; } } - + /** * Ingest job mode. */ @@ -71,14 +71,14 @@ public final class IngestJob { BATCH, STREAMING } - + private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); private final static AtomicLong nextId = new AtomicLong(0L); private final long id; private final List dataSources = new ArrayList<>(); private final List files = new ArrayList<>(); private final Mode ingestMode; - private final Map ingestJobPipelines; + private final Map ingestJobPipelines; private final AtomicInteger incompleteJobsCount; private final IngestJobSettings settings; private volatile CancellationReason cancellationReason; @@ -113,14 +113,14 @@ public final class IngestJob { this(Arrays.asList(dataSource), settings); this.files.addAll(files); } - + /** - * Constructs an ingest job that analyzes one data source, possibly using - * an ingest stream. + * Constructs an ingest job that analyzes one data source, possibly using an + * ingest stream. * - * @param settings The ingest job settings. + * @param settings The ingest job settings. */ - IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) { + IngestJob(Content dataSource, Mode ingestMode, IngestJobSettings settings) { this.id = IngestJob.nextId.getAndIncrement(); this.ingestJobPipelines = new ConcurrentHashMap<>(); this.dataSources.add(dataSource); @@ -149,10 +149,10 @@ public final class IngestJob { boolean hasIngestPipeline() { return (!settings.getEnabledIngestModuleTemplates().isEmpty()); } - + /** * Add a set of files (by object ID) to be ingested. - * + * * @param fileObjIds the list of file IDs */ void addStreamingIngestFiles(List fileObjIds) { @@ -161,10 +161,10 @@ public final class IngestJob { return; } // Streaming ingest jobs will only have one data source - IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); + IngestPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); streamingIngestPipeline.addStreamingIngestFiles(fileObjIds); } - + /** * Start data source processing for streaming ingest. */ @@ -174,8 +174,8 @@ public final class IngestJob { return; } // Streaming ingest jobs will only have one data source - IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); - streamingIngestPipeline.processStreamingIngestDataSource(); + IngestPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); + streamingIngestPipeline.addStreamingIngestDataSource(); } /** @@ -185,31 +185,27 @@ public final class IngestJob { * @return A collection of ingest module start up errors, empty on success. */ List start() { - + /* - * Set up the pipeline(s) + * Set up the ingest job pipelines, one for each data source to be + * ingested by this job. */ if (files.isEmpty()) { for (Content dataSource : dataSources) { - IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSource, settings); - this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); + IngestPipeline ingestJobPipeline = new IngestPipeline(this, dataSource, settings); + ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); } } else { - IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSources.get(0), files, settings); - this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); + IngestPipeline ingestJobPipeline = new IngestPipeline(this, dataSources.get(0), files, settings); + ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); } incompleteJobsCount.set(ingestJobPipelines.size()); - + /* - * Try to start each data source ingest job. Note that there is an - * assumption here that if there is going to be a module - * startup failure, it will be for the first ingest job pipeline. - * - * TODO (RC): Consider separating module start up from pipeline startup - * so that no processing is done if this assumption is false. + * Try to start up each ingest job pipeline. Stop at the first failure. */ List errors = new ArrayList<>(); - for (IngestJobPipeline ingestJobPipeline : this.ingestJobPipelines.values()) { + for (IngestPipeline ingestJobPipeline : ingestJobPipelines.values()) { errors.addAll(ingestJobPipeline.startUp()); if (errors.isEmpty() == false) { break; @@ -220,8 +216,8 @@ public final class IngestJob { * Handle start up success or failure. */ if (errors.isEmpty()) { - for (IngestJobPipeline dataSourceJob : this.ingestJobPipelines.values()) { - IngestManager.getInstance().fireDataSourceAnalysisStarted(id, dataSourceJob.getId(), dataSourceJob.getDataSource()); + for (IngestPipeline ingestJobPipeline : ingestJobPipelines.values()) { + IngestManager.getInstance().fireDataSourceAnalysisStarted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource()); } } else { cancel(CancellationReason.INGEST_MODULES_STARTUP_FAILED); @@ -229,14 +225,14 @@ public final class IngestJob { return errors; } - + /** * Get the ingest mode for this job (batch or streaming). - * + * * @return the ingest mode. */ Mode getIngestMode() { - return ingestMode; + return ingestMode; } /** @@ -251,8 +247,8 @@ public final class IngestJob { /** * Gets a snapshot of the progress of this ingest job. * - * @param getIngestTasksSnapshot - * + * @param getIngestTasksSnapshot + * * @return The snapshot. */ public ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot) { @@ -326,7 +322,7 @@ public final class IngestJob { * * @param ingestJobPipeline A completed ingestJobPipeline. */ - void ingestJobPipelineFinished(IngestJobPipeline ingestJobPipeline) { + void notifyIngestPipelineShutDown(IngestPipeline ingestJobPipeline) { IngestManager ingestManager = IngestManager.getInstance(); if (!ingestJobPipeline.isCancelled()) { ingestManager.fireDataSourceAnalysisCompleted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource()); @@ -412,7 +408,7 @@ public final class IngestJob { fileIngestRunning = false; fileIngestStartTime = null; dataSourceProcessingSnapshots = new ArrayList<>(); - for (IngestJobPipeline pipeline : ingestJobPipelines.values()) { + for (IngestPipeline pipeline : ingestJobPipelines.values()) { Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot); dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot)); if (null == dataSourceModule) { @@ -499,7 +495,7 @@ public final class IngestJob { */ public static class DataSourceIngestModuleHandle { - private final IngestJobPipeline ingestJobPipeline; + private final IngestPipeline ingestJobPipeline; private final DataSourceIngestPipeline.DataSourcePipelineModule module; private final boolean cancelled; @@ -508,10 +504,11 @@ public final class IngestJob { * used to get basic information about the module and to request * cancellation of the module. * - * @param ingestJobPipeline The ingestJobPipeline that owns the data source level ingest module. - * @param module The data source level ingest module. + * @param ingestJobPipeline The ingestJobPipeline that owns the data + * source level ingest module. + * @param module The data source level ingest module. */ - private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) { + private DataSourceIngestModuleHandle(IngestPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) { this.ingestJobPipeline = ingestJobPipeline; this.module = module; this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 697e512ad8..2c3f1e556e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -28,9 +28,9 @@ import org.sleuthkit.datamodel.Content; */ public final class IngestJobContext { - private final IngestJobPipeline ingestJobPipeline; + private final IngestPipeline ingestJobPipeline; - IngestJobContext(IngestJobPipeline ingestJobPipeline) { + IngestJobContext(IngestPipeline ingestJobPipeline) { this.ingestJobPipeline = ingestJobPipeline; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java similarity index 61% rename from Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java rename to Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java index afcf1f2eb9..e3d04eb4bb 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java @@ -52,67 +52,108 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.modules.interestingitems.FilesSet; import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer; -import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.DataArtifact; +import org.sleuthkit.datamodel.DataSource; /** - * A set of ingest module pipelines for an ingest job. + * An ingest pipeline that works with the ingest tasks scheduler to coordinate + * the creation, scheduling, and execution of ingest tasks for one of the data + * sources in an ingest job. An ingest pipeline has one to many child ingest + * task pipelines. An ingest task pipeline is a sequences of ingest modules of a + * given type that have been enabled and configured as part of the settings for + * the ingest job. */ -final class IngestJobPipeline { +final class IngestPipeline { private static String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy"; - // to match something like: "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" + /* + * The class names of the proxy classes Jython generates for Python classes + * look something like: + * "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" + */ private static final Pattern JYTHON_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); - private static final Logger logger = Logger.getLogger(IngestJobPipeline.class.getName()); - - private final IngestJob job; + private static final Logger logger = Logger.getLogger(IngestPipeline.class.getName()); private static final AtomicLong nextPipelineId = new AtomicLong(0L); - private final long pipelineId; - private final IngestJobSettings settings; - private Content dataSource; - private final IngestJob.Mode ingestMode; - private final List files = new ArrayList<>(); - private final long createTime; + + /** + * An ingest pipeline interacts with the ingest task scheduler to schedule + * initial ingest tasks, determine whether or not there are ingest tasks + * still to be executed, and to schedule additional tasks submitted by + * ingest modules. For example, a file carving module can add carved files + * to an ingest job and many modules will add data artifacts to an ingest + * job. + */ + private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); /* - * An ingest job pipeline runs in stages. + * An ingest pipeline runs its child ingest task pipelines in stages. */ private static enum Stages { /* - * Setting up for processing. + * The ingest pipeline is setting up its child ingest task pipelines. */ INITIALIZATION, /* - * Running only file and data artifact ingest modules (used only for - * streaming ingest). + * The ingest pipeline is only running its ingest task pipelines that + * process results such as data artifacts, OS accounts, etc., that are + * generated by some other part of the ingest process. Typically, but + * not always, this will be a transitional stage that happens first + * because the results pipelines need to be started up before the file + * and data source level pipelines, so that results generated by those + * pipelines can be processed by the results pipelines. */ - FIRST_STAGE_FILES_ONLY, + FIRST_STAGE_RESULTS_TASKS_ONLY, /* - * Running high priority data source level ingest modules, file ingest - * modules and data artifact ingest modules. + * The ingest pipeline is only running its file and results ingest task + * pipelines because the data source for the job has not been supplied + * to it yet. This stage is only used for streaming ingest. */ - FIRST_STAGE_FILES_AND_DATASOURCE, + FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY, + /* + * The ingest pipeline is running its high priority data source level + * ingest task pipeline, its file ingest pipeline, and its results + * pipelines. + */ + FIRST_STAGE_ALL_TASKS, /** - * Running lower priority, usually long-running, data source level - * ingest modules and data artifact ingest modules. + * The ingest pipeline is running its lower priority data source level + * ingest task pipeline and its results task pipelines. */ SECOND_STAGE, /** - * Cleaning up. + * The ingest pipeline is shutting down its child ingest task pipelines. */ FINALIZATION }; - private volatile Stages stage = IngestJobPipeline.Stages.INITIALIZATION; - private final Object stageCompletionCheckLock = new Object(); + + private final IngestJob job; + private final long pipelineId; + private final IngestJobSettings settings; + private DataSource dataSource; + private final IngestJob.Mode ingestMode; + private final List files = new ArrayList<>(); + private final long createTime; + private volatile Stages stage = IngestPipeline.Stages.INITIALIZATION; + private final Object stageTransitionLock = new Object(); + + /* + * An ingest pipeline has at most a single data artifact ingest task + * pipeline. + * + * The pipeline may be empty, depending on which modules are enabled in the + * ingest job settings. + */ + private DataArtifactIngestPipeline artifactIngestPipeline; /** - * An ingest job pipeline has separate data source level ingest task - * pipelines for the first and second processing stages. Longer running, - * lower priority modules belong in the second stage pipeline, although this - * cannot be enforced. Note that the pipelines for both stages are created - * at job start up to allow for verification that they both can be started - * up without errors. + * An ingest pipeline has separate data source level ingest task pipelines + * for the first and second processing stages. Longer running, lower + * priority modules belong in the second stage pipeline, although this + * cannot be enforced. + * + * Either or both pipelines may be empty, depending on which modules are + * enabled in the ingest job settings. */ private final Object dataSourceIngestPipelineLock = new Object(); private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; @@ -120,24 +161,22 @@ final class IngestJobPipeline { private DataSourceIngestPipeline currentDataSourceIngestPipeline; /** - * An ingest job pipeline has a collection of identical file ingest task + * An ingest pipeline has a collection of identical file ingest task * pipelines, one for each file ingest thread in the ingest manager. The - * ingest threads take and return task pipelines using a blocking queue. A - * fixed list of all of the task pipelines is used to allow the job pipeline - * to cycle through the individual task pipelines to check their status and - * their progress on their current tasks. + * ingest threads take and return task pipelines using a blocking queue. + * Additionally, a fixed list of all of the file pipelines is used to allow + * cycling through each of the individual task pipelines to check their + * status. + * + * The pipelines may be empty, depending on which modules are enabled in the + * ingest job settings. */ private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); private final List fileIngestPipelines = new ArrayList<>(); - /* - * An ingest job pipeline has a single data aritfact task pipeline. - */ - private DataArtifactIngestPipeline artifactIngestPipeline; - /** - * An ingest job pipeline supports cancellation of either the currently - * running data source level ingest module or the entire ingest job. + * An ingest pipeline supports cancellation of either the currently running + * data source level ingest pipeline or all of its child pipelines. * Cancellation works by setting flags that are checked by the ingest task * pipelines every time they transition from from one module to another. * Modules are also expected to check these flags and stop processing if @@ -150,22 +189,15 @@ final class IngestJobPipeline { private volatile IngestJob.CancellationReason cancellationReason = IngestJob.CancellationReason.NOT_CANCELLED; /** - * An ingest job pipeline interacts with the ingest task scheduler to - * determine whether or not there are ingest tasks still to be executed for - * an ingest job and to schedule additional tasks submitted by ingest - * modules. For example, a file carving module can add carved files to an - * ingest job and many modules will add data artifacts to an ingest job. - */ - private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - - /** - * If running in a GUI, an ingest job pipeline report progress and allows a - * user to cancel a data source level ingest module or the entire ingest job - * using progress bars in the lower right hand corner of the main - * application window. This is one of a handful of things that currently - * couples the ingest code to the presentation layer. + * If running in a GUI, an ingest pipeline reports progress and allows a + * user to cancel either an individual data source level ingest module or + * all of its ingest tasks using progress bars in the lower right hand + * corner of the main application window. There is also support for ingest + * process snapshots and recording ingest job details in the case database. */ private final boolean doUI; + private final Object resultsIngestProgressLock = new Object(); + private ProgressHandle resultsIngestProgress; private final Object dataSourceIngestProgressLock = new Object(); private ProgressHandle dataSourceIngestProgress; private final Object fileIngestProgressLock = new Object(); @@ -176,22 +208,31 @@ final class IngestJobPipeline { private String currentFileIngestModule = ""; private String currentFileIngestTask = ""; private final List ingestModules = new ArrayList<>(); - private volatile IngestJobInfo ingestJob; + private volatile IngestJobInfo ingestJobInfo; /** - * Constructs a set of ingest module pipelines for an ingest job. + * Constructs an ingest pipeline that works with the ingest tasks scheduler + * to coordinate the creation, scheduling, and execution of ingest tasks for + * one of the data sources in an ingest job. An ingest pipeline has one to + * many child ingest task pipelines. The child ingest task pipelines are + * sequences of ingest modules of a given type that have been enabled and + * configured as part of the settings for the ingest job. * * @param job The ingest job. - * @param dataSource The data source for the ingest job. + * @param dataSource The data source. * @param settings The ingest settings for the ingest job. */ - IngestJobPipeline(IngestJob job, Content dataSource, IngestJobSettings settings) { + IngestPipeline(IngestJob job, Content dataSource, IngestJobSettings settings) { this(job, dataSource, Collections.emptyList(), settings); } /** - * Constructs a set of ingest module pipelines for an ingest job that - * process a subset of the files in the job's data source. + * Constructs an ingest pipeline that works with the ingest tasks scheduler + * to coordinate the creation, scheduling, and execution of ingest tasks for + * one of the data sources in an ingest job. An ingest pipeline has one to + * many child ingest task pipelines. The child ingest task pipelines are + * sequences of ingest modules of a given type that have been enabled and + * configured as part of the settings for the ingest job. * * @param job The ingest job. * @param dataSource The data source for the ingest job. @@ -200,18 +241,21 @@ final class IngestJobPipeline { * processed. * @param settings The ingest settings for the ingest job. */ - IngestJobPipeline(IngestJob job, Content dataSource, List files, IngestJobSettings settings) { + IngestPipeline(IngestJob job, Content dataSource, List files, IngestJobSettings settings) { + if (!(dataSource instanceof DataSource)) { + throw new IllegalArgumentException("Passed dataSource that does not implement the DataSource interface"); //NON-NLS + } this.job = job; - this.dataSource = dataSource; + this.dataSource = (DataSource) dataSource; this.settings = settings; this.files.addAll(files); - pipelineId = IngestJobPipeline.nextPipelineId.getAndIncrement(); + pipelineId = IngestPipeline.nextPipelineId.getAndIncrement(); ingestMode = job.getIngestMode(); doUI = RuntimeProperties.runningWithGUI(); createTime = new Date().getTime(); stage = Stages.INITIALIZATION; - currentFileIngestModule = ""; - currentFileIngestTask = ""; + currentFileIngestModule = ""; //NON-NLS + currentFileIngestTask = ""; //NON-NLS createIngestTaskPipelines(); } @@ -240,9 +284,11 @@ final class IngestJobPipeline { } /** - * Takes a Jython proxy class name like + * Extracts a module class name from a Jython module proxy class name. For + * example, a Jython class name such * "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" - * and returns "GPX_Parser_Module.GPXParserFileIngestModuleFactory." + * will be parsed to return + * "GPX_Parser_Module.GPXParserFileIngestModuleFactory." * * @param className The class name. * @@ -251,7 +297,7 @@ final class IngestJobPipeline { private static String getModuleNameFromJythonClassName(String className) { Matcher m = JYTHON_REGEX.matcher(className); if (m.find()) { - return String.format("%s.%s", m.group(1), m.group(2)); + return String.format("%s.%s", m.group(1), m.group(2)); //NON-NLS } else { return null; } @@ -278,7 +324,7 @@ final class IngestJobPipeline { } /** - * Creates the ingest task pipelines for this ingest job pipeline. + * Creates the child ingest task pipelines for this ingest pipeline. */ private void createIngestTaskPipelines() { /* @@ -322,9 +368,9 @@ final class IngestJobPipeline { * for each ingest task pipeline. */ IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, unorderedJythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + List firstStageDataSourceModuleTemplates = IngestPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, unorderedJythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); + List fileIngestModuleTemplates = IngestPipeline.getConfiguredIngestModuleTemplates(unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); + List secondStageDataSourceModuleTemplates = IngestPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); /** * Add any module templates that were not specified in the pipeline @@ -336,7 +382,7 @@ final class IngestJobPipeline { addOrdered(firstStageDataSourceModuleTemplates, unorderedDataSourceModuleTemplates, unorderedJythonDataSourceModuleTemplates); addOrdered(fileIngestModuleTemplates, unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates); addOrdered(fileIngestModuleTemplates, unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates); - List artifactModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + List artifactModuleTemplates = IngestPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); /** * Construct the ingest task pipelines from the ordered lists. @@ -370,14 +416,14 @@ final class IngestJobPipeline { addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase); addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); } catch (TskCoreException | NoCurrentCaseException ex) { - logErrorMessage(Level.WARNING, "Failed to add ingest modules listing to case database", ex); + logErrorMessage(Level.WARNING, "Failed to add ingest modules listing to case database", ex); //NON-NLS } } /** * Adds a list of ingest module templates for an ingest task pipeline to the - * "master list" and records the final composition of the pipeline in the - * case database. + * modules "master list" and records the final composition of the pipeline + * in the case database. * * @param templates A list of ingest module templates. * @param type The type of the ingest module templates. @@ -425,9 +471,9 @@ final class IngestJobPipeline { } /** - * Gets the ID of this ingest job pipeline. + * Gets the ID of this ingest pipeline. * - * @return The job ID. + * @return The ID. */ long getId() { return this.pipelineId; @@ -443,17 +489,17 @@ final class IngestJobPipeline { } /** - * Getss the data source to be ingested by this ingest job pipeline. + * Gets the data source to be ingested by this ingest pipeline. * - * @return A Content object representing the data source. + * @return The data source. */ - Content getDataSource() { - return this.dataSource; + DataSource getDataSource() { + return dataSource; } /** * Queries whether or not unallocated space should be processed by this - * ingest job pipeline. + * ingest pipeline. * * @return True or false. */ @@ -462,8 +508,7 @@ final class IngestJobPipeline { } /** - * Gets the file ingest filter for this ingest job pipeline from the ingest - * job settings. + * Gets the file ingest filter for this ingest pipeline. * * @return The filter. */ @@ -472,89 +517,84 @@ final class IngestJobPipeline { } /** - * Checks to see if this ingest job pipeline has at least one ingest task - * pipeline. + * Checks to see if this ingest pipeline has at least one ingest module to + * run. * * @return True or false. */ - boolean hasIngestPipeline() { - return hasFirstStageDataSourceIngestPipeline() - || hasFileIngestPipeline() - || hasSecondStageDataSourceIngestPipeline() - || hasDataArtifactPipeline(); + boolean hasIngestModules() { + return hasArtifactIngestModules() + || hasFileIngestModules() + || hasFirstStageDataSourceIngestModules() + || hasSecondStageDataSourceIngestModules(); } /** - * Checks to see if this ingest job pipeline has a first stage data source - * level ingest pipeline. + * Checks to see if this ingest pipeline has at least one data artifact + * ingest module. * * @return True or false. */ - private boolean hasFirstStageDataSourceIngestPipeline() { - return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); + private boolean hasArtifactIngestModules() { + return (artifactIngestPipeline.isEmpty() == false); } /** - * Checks to see if this ingest job pipeline has a second stage data source - * level ingest task pipeline. + * Checks to see if this ingest pipeline has at least one first stage data + * source level ingest modules. * * @return True or false. */ - private boolean hasSecondStageDataSourceIngestPipeline() { - return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); + private boolean hasFirstStageDataSourceIngestModules() { + return (firstStageDataSourceIngestPipeline.isEmpty() == false); } /** - * Checks to see if this ingest job pipeline has a file level ingest task - * pipeline. + * Checks to see if this ingest pipeline has at least one second stage data + * source level ingest modules. * * @return True or false. */ - private boolean hasFileIngestPipeline() { - if (!this.fileIngestPipelines.isEmpty()) { - return !this.fileIngestPipelines.get(0).isEmpty(); + private boolean hasSecondStageDataSourceIngestModules() { + return (secondStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if this ingest pipeline has at least one file ingest + * module. + * + * @return True or false. + */ + private boolean hasFileIngestModules() { + if (!fileIngestPipelines.isEmpty()) { + /* + * Note that the file ingest task pipelines are identical. + */ + return !fileIngestPipelines.get(0).isEmpty(); } return false; } /** - * Checks to see if this ingest job pipeline has a data artifact ingest task - * pipeline. - * - * @return True or false. - */ - private boolean hasDataArtifactPipeline() { - return (artifactIngestPipeline.isEmpty() == false); - } - - /** - * Starts up this ingest job pipeline. + * Starts up this ingest pipeline. * * @return A collection of ingest module startup errors, empty on success. */ List startUp() { if (dataSource == null) { - throw new IllegalStateException("Ingest started before setting data source"); + throw new IllegalStateException("Ingest started before setting data source"); //NON-NLS } + List errors = startUpIngestTaskPipelines(); if (errors.isEmpty()) { - try { - ingestJob = Case.getCurrentCaseThrows().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); - } catch (TskCoreException | NoCurrentCaseException ex) { - logErrorMessage(Level.WARNING, "Failed to add ingest job info to case database", ex); //NON-NLS - } - - startArtifactProcessing(); - if (hasFirstStageDataSourceIngestPipeline() || hasFileIngestPipeline()) { + recordIngestJobStartUpInfo(); + if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules()) { if (ingestMode == IngestJob.Mode.BATCH) { - logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS - startFirstStage(); + startFirstStageInBatchMode(); } else { - logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS - startFileIngestStreaming(); + startFirstStageInStreamingMode(); } - } else if (hasSecondStageDataSourceIngestPipeline()) { - logInfoMessage("Starting second stage analysis"); //NON-NLS + } else if (hasSecondStageDataSourceIngestModules()) { startSecondStage(); } } @@ -562,38 +602,58 @@ final class IngestJobPipeline { } /** - * Starts up each of the ingest task pipelines in this ingest job pipeline. - * All of the pipelines are started so that any and all start up errors can - * be returned to the caller. It is important to capture all of the errors - * since the ingest job will be automatically cancelled and the errors will - * be reported to the user so the issues can be addressed or the modules - * that can't start up can be disabled before the ingest job is attempted - * again. + * Writes start up data about the ingest job into the case database. The + * case database returns a an object that is retained to allow the additon + * of a completion time when the ingest job is finished. + * + * This is here, rather than in the IngestJob class, because the logged data + * includes ordered lists of the ingest modules in the child ingest task + * pipelines. + */ + void recordIngestJobStartUpInfo() { + try { + SleuthkitCase caseDb = Case.getCurrentCaseThrows().getSleuthkitCase(); + ingestJobInfo = caseDb.addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); + } catch (TskCoreException | NoCurrentCaseException ex) { + logErrorMessage(Level.WARNING, "Failed to add ingest job info to case database", ex); //NON-NLS + } + } + + /** + * Starts up each of the child ingest task pipelines in this ingest + * pipeline. + * + * Note that all of the child pipelines are started so that any and all + * start up errors can be returned to the caller. It is important to capture + * all of the errors, because the ingest job will be automatically cancelled + * and the errors will be reported to the user so either the issues can be + * addressed or the modules that can't start up can be disabled before the + * ingest job is attempted again. * * @return A list of ingest module startup errors, empty on success. */ private List startUpIngestTaskPipelines() { List errors = new ArrayList<>(); - startUpIngestTaskPipeline(firstStageDataSourceIngestPipeline); - startUpIngestTaskPipeline(secondStageDataSourceIngestPipeline); + errors.addAll(startUpIngestTaskPipeline(firstStageDataSourceIngestPipeline)); + errors.addAll(startUpIngestTaskPipeline(secondStageDataSourceIngestPipeline)); for (FileIngestPipeline pipeline : fileIngestPipelines) { List filePipelineErrors = startUpIngestTaskPipeline(pipeline); if (!filePipelineErrors.isEmpty()) { /* - * If one pipeline copy can't start up, assume that none of them - * will be able to start up for the same reasons. + * If one file pipeline copy can't start up, assume that none of + * them will be able to start up for the same reason. */ errors.addAll(filePipelineErrors); break; } } - errors.addAll(this.artifactIngestPipeline.startUp()); + errors.addAll(startUpIngestTaskPipeline(artifactIngestPipeline)); return errors; } /** * Starts up an ingest task pipeline. If there are any start up errors, the - * piepline is imediately shut down. + * pipeline is imediately shut down. * * @param pipeline The ingest task pipeline to start up. * @@ -608,6 +668,9 @@ final class IngestJobPipeline { * The start up errors will ultimately be reported to the user * for possible remedy, but the shut down errors are logged * here. + * + * RJCTODO: Is this right? What is our contract with the module + * developers when start up fails? */ logIngestModuleErrors(shutDownErrors); } @@ -616,123 +679,163 @@ final class IngestJobPipeline { } /** - * Schedules data artifact tasks for all of the existing data artifacts for - * the ingest job's data source, if any. Note that there is a possiblity, - * just as in with the other item types, that a user may process artifacts - * more than once if the same ingest job is run more than once. + * Schedules the first stage results, data source level, and file ingest + * tasks for the data source. */ - private void startArtifactProcessing() { - Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); - try { - List artifacts = blackboard.getDataArtifacts(dataSource); - taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); - } catch (TskCoreException ex) { - // RJCTODO - } - } + private void startFirstStageInBatchMode() { + logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS + stage = Stages.FIRST_STAGE_ALL_TASKS; - /** - * Starts the first stage of the ingest job. - */ - private void startFirstStage() { - this.stage = IngestJobPipeline.Stages.FIRST_STAGE_FILES_AND_DATASOURCE; - - if (this.hasFileIngestPipeline()) { - synchronized (this.fileIngestProgressLock) { - this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); - } - } - - if (this.doUI) { - /** - * Start one or both of the first stage ingest progress bars. + if (hasFileIngestModules()) { + /* + * The estimated number of files remaining to be processed is used + * for ingest snapshots and for the file ingest progress bar. */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); - } - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); - } - } - - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - - /** - * Schedule the first stage tasks. - */ - if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { - logInfoMessage("Scheduling first stage data source and file level analysis tasks"); //NON-NLS - IngestJobPipeline.taskScheduler.scheduleIngestTasks(this); - } else if (this.hasFirstStageDataSourceIngestPipeline()) { - logInfoMessage("Scheduling first stage data source level analysis tasks"); //NON-NLS - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - logInfoMessage("Scheduling file level analysis tasks, no first stage data source level analysis configured"); //NON-NLS - IngestJobPipeline.taskScheduler.scheduleFileIngestTasks(this, this.files); - - /** - * No data source ingest task has been scheduled for this stage, and - * it is possible, if unlikely, that no file ingest tasks were - * actually scheduled since there are files that get filtered out by - * the tasks scheduler. In this special case, an ingest thread will - * never get to check for completion of this stage of the job, so do - * it now. - */ - this.checkForStageCompleted(); - } - } - - /** - * Prepares for file ingest. Used for streaming ingest. Does not schedule - * any file tasks - those will come from calls to addStreamingIngestFiles(). - */ - private void startFileIngestStreaming() { - synchronized (this.stageCompletionCheckLock) { - this.stage = IngestJobPipeline.Stages.FIRST_STAGE_FILES_ONLY; - } - - if (this.hasFileIngestPipeline()) { - synchronized (this.fileIngestProgressLock) { - this.estimatedFilesToProcess = 0; // Set to indeterminate until the data source is complete - } - } - - if (this.doUI) { - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); - } - } - - logInfoMessage("Waiting for streaming files"); //NON-NLS - } - - /** - * Start data source ingest. Used for streaming ingest when the data source - * is not ready when ingest starts. - */ - private void startDataSourceIngestStreaming() { - - // Now that the data source is complete, we can get the estimated number of - // files and switch to a determinate progress bar. - synchronized (fileIngestProgressLock) { - if (null != this.fileIngestProgress) { + synchronized (fileIngestProgressLock) { estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); - fileIngestProgress.switchToDeterminate((int) estimatedFilesToProcess); } } - if (this.doUI) { - /** - * Start the first stage data source ingest progress bar. + /* + * If running with a GUI, start ingest progress bars in the lower right + * hand corner of the main application window. + */ + if (doUI) { + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + if (hasFileIngestModules()) { + startFileIngestProgressBar(); + } + if (hasArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } + } + + /** + * Make the first stage data source level ingest pipeline the current + * data source level pipeline. + */ + synchronized (dataSourceIngestPipelineLock) { + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + } + + /** + * Schedule the first stage ingest tasks. + */ + if (hasArtifactIngestModules()) { + /* + * Create ingest tasks for data artifacts already in the case + * database. Additional tasks will be created as other ingest + * modules add data aritfacts. */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); + taskScheduler.scheduleDataArtifactIngestTasks(this); + } + if (hasFirstStageDataSourceIngestModules() && hasFileIngestModules()) { + taskScheduler.scheduleDataSourceAndFileIngestTasks(this); + } else if (hasFirstStageDataSourceIngestModules()) { + taskScheduler.scheduleDataSourceIngestTask(this); + } else if (hasFileIngestModules() && !files.isEmpty()) { + taskScheduler.scheduleFileIngestTasks(this, files); + /** + * No data source ingest task has been scheduled for this stage, it + * is possible that no artifact tasks were scheduled, and it is also + * possible that no file ingest tasks were scheduled when the task + * scheduler applied the file ingest filter. In this special case in + * which there are no ingest tasks to do, an ingest thread will + * never get to check for completion of this stage of the job, so do + * it now so there is not a do-nothing ingest job that lives + * forever. + */ + checkForStageCompleted(); + } + } + + /** + * Schedules the first stage results ingest tasks and prepares for streaming + * file ingest (used for streaming ingest only). Does not schedule any file + * tasks - those will come from calls to addStreamingIngestFiles(). + */ + private void startFirstStageInStreamingMode() { + logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS + stage = Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY; + + if (hasFileIngestModules()) { + /* + * The estimated number of files remaining to be processed is used + * for ingest snapshots and for the file ingest progress bar. + * However, for streaming ingest, it cannot be calculated until the + * data source is added to this pipeline. Start with zero to signal + * an unknown value. + */ + synchronized (fileIngestProgressLock) { + estimatedFilesToProcess = 0; + } + } + + /* + * If running with a GUI, start ingest progress bars in the lower right + * hand corner of the main application window. + */ + if (doUI) { + if (hasArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } + if (hasFileIngestModules()) { + /* + * Note that because estimated files remaining to process has + * been set to zero, the progress bar will start in the + * "indeterminate" state. + */ + startFileIngestProgressBar(); + } + } + + /** + * Schedule the first stage ingest tasks for streaming mode. For + * streaming ingest, file ingest tasks are created as the files are + * streamed into this pipeline, so there are no tasks to schedule yet. + * Also, for streaming ingest, the data source is not added to the + * pipeline right away, so there is no data source level task to + * schedule yet. So only result ingest tasks can be scheduled at this + * time. + */ + if (hasArtifactIngestModules()) { + /* + * Create ingest tasks for data artifacts already in the case + * database. Additional tasks will be created as other ingest + * modules add data aritfacts. + */ + taskScheduler.scheduleDataArtifactIngestTasks(this); + } + } + + /** + * Starts a data source level ingest. Used for streaming ingest, in which + * the data source is not ready when ingest starts. + */ + private void startDataSourceTaskInStreamingMode() { + /* + * Now that the data source processor analysis of the data source is + * complete, an estimate of the files remaining to be processed can be + * calculated and the file ingest progress bar in the lower right hand + * corner of the main application window Fcan be switched from + * indeterminate to determinate. + */ + synchronized (fileIngestProgressLock) { + estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); + if (doUI && fileIngestProgress != null) { + fileIngestProgress.switchToDeterminate((int) estimatedFilesToProcess); // RJCTODO: Check assumption that bar start indeterminate + } + } + + if (doUI) { + /** + * Start the first stage data source ingest progress bar in the + * lower right hand corner of the main application window. + */ + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); } } @@ -744,27 +847,46 @@ final class IngestJobPipeline { this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; } - logInfoMessage("Scheduling first stage data source level analysis tasks"); //NON-NLS - synchronized (this.stageCompletionCheckLock) { - this.stage = IngestJobPipeline.Stages.FIRST_STAGE_FILES_AND_DATASOURCE; - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); + logInfoMessage("Scheduling first stage data source level ingest task in streaming mode"); //NON-NLS + synchronized (this.stageTransitionLock) { + stage = IngestPipeline.Stages.FIRST_STAGE_ALL_TASKS; + IngestPipeline.taskScheduler.scheduleDataSourceIngestTask(this); } } /** - * Starts the second stage of this ingest job. + * Starts the second stage ingest task pipelines. */ private void startSecondStage() { - logInfoMessage("Starting second stage analysis"); //NON-NLS - this.stage = IngestJobPipeline.Stages.SECOND_STAGE; - if (this.doUI) { - this.startDataSourceIngestProgressBar(); + logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), job.getId())); //NON-NLS + stage = IngestPipeline.Stages.SECOND_STAGE; + if (doUI) { + startDataSourceIngestProgressBar(); } - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + synchronized (dataSourceIngestPipelineLock) { + currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; + } + taskScheduler.scheduleDataSourceIngestTask(this); + } + + /** + * Starts a progress bar for the results ingest tasks for the ingest job. + */ + private void startArtifactIngestProgressBar() { + if (doUI) { + synchronized (resultsIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()); // RJCTODO + resultsIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + IngestPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + return true; + } + }); + resultsIngestProgress.start(); + resultsIngestProgress.switchToIndeterminate(); + } } - logInfoMessage("Scheduling second stage data source level analysis tasks"); //NON-NLS - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); } /** @@ -786,12 +908,12 @@ final class IngestJobPipeline { // the user wants to cancel only the currently executing // data source ingest module or the entire ingest job. DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(IngestJobPipeline.this.getClass(), "IngestJob.cancellationDialog.title"); + String dialogTitle = NbBundle.getMessage(IngestPipeline.this.getClass(), "IngestJob.cancellationDialog.title"); JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); if (panel.cancelAllDataSourceIngestModules()) { - IngestJobPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + IngestPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); } else { - IngestJobPipeline.this.cancelCurrentDataSourceIngestModule(); + IngestPipeline.this.cancelCurrentDataSourceIngestModule(); } return true; } @@ -818,7 +940,7 @@ final class IngestJobPipeline { // the cancel button on the progress bar and the OK button // of a cancelation confirmation dialog supplied by // NetBeans. - IngestJobPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + IngestPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); return true; } }); @@ -836,7 +958,7 @@ final class IngestJobPipeline { if (ingestMode == IngestJob.Mode.BATCH) { checkForStageCompletedBatch(); } else { - checkForStageCompletedStreaming(); + checkForStageCompletedInStreamingMode(); } } @@ -845,14 +967,16 @@ final class IngestJobPipeline { * completed and does a stage transition if they are. */ private void checkForStageCompletedBatch() { - synchronized (this.stageCompletionCheckLock) { - if (IngestJobPipeline.taskScheduler.currentTasksAreCompleted(this)) { + // RJCTODO: This logic depends on carved/derived files for a task being + // added before the task is completed. See addFiles(). + synchronized (this.stageTransitionLock) { + if (IngestPipeline.taskScheduler.currentTasksAreCompleted(this)) { switch (this.stage) { - case FIRST_STAGE_FILES_AND_DATASOURCE: + case FIRST_STAGE_ALL_TASKS: this.finishFirstStage(); break; case SECOND_STAGE: - this.finish(); + this.shutDown(); break; } } @@ -863,19 +987,21 @@ final class IngestJobPipeline { * Checks to see if the ingest tasks for the current stage of this job are * completed and does a stage transition if they are. */ - private void checkForStageCompletedStreaming() { - synchronized (this.stageCompletionCheckLock) { - if (IngestJobPipeline.taskScheduler.currentTasksAreCompleted(this)) { + private void checkForStageCompletedInStreamingMode() { + // RJCTODO: This logic depends on carved/derived files for a task being + // added before the task is completed. See addFiles(). + synchronized (this.stageTransitionLock) { + if (IngestPipeline.taskScheduler.currentTasksAreCompleted(this)) { switch (this.stage) { - case FIRST_STAGE_FILES_ONLY: + case FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY: // Nothing to do here - need to wait for the data source break; - case FIRST_STAGE_FILES_AND_DATASOURCE: + case FIRST_STAGE_ALL_TASKS: // Finish file and data source ingest, start second stage (if applicable) this.finishFirstStage(); break; case SECOND_STAGE: - this.finish(); + this.shutDown(); break; } } @@ -923,67 +1049,73 @@ final class IngestJobPipeline { } } + // RJCTODO: Shut down the data source level pipeline /* * Start the second stage, if appropriate. */ - if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); + if (!this.cancelled && this.hasSecondStageDataSourceIngestModules()) { + startSecondStage(); } else { - this.finish(); + shutDown(); } } /** * Shuts down the ingest pipelines and progress bars for this job. */ - private void finish() { - logInfoMessage("Finished analysis"); //NON-NLS - this.stage = IngestJobPipeline.Stages.FINALIZATION; + private void shutDown() { + logInfoMessage("Finished all tasks"); //NON-NLS + stage = IngestPipeline.Stages.FINALIZATION; - if (this.doUI) { + // RJCTODO: SHould all the progress bars be attended to? + if (doUI) { // Finish the second stage data source ingest progress bar, if it hasn't // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgress != null) { + dataSourceIngestProgress.finish(); + dataSourceIngestProgress = null; } } } - if (ingestJob != null) { - if (this.cancelled) { + + // RJCTODO: Shut down pipelines + // RJCTODO: Put in a method + if (ingestJobInfo != null) { + if (cancelled) { try { - ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED); + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); } catch (TskCoreException ex) { logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); } } else { try { - ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED); + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); } catch (TskCoreException ex) { logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); } } try { - this.ingestJob.setEndDateTime(new Date()); + ingestJobInfo.setEndDateTime(new Date()); } catch (TskCoreException ex) { logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } } - this.job.ingestJobPipelineFinished(this); + + job.notifyIngestPipelineShutDown(this); } /** - * Executes an ingest task for an ingest job by passing the item associated - * with the task through the appropriate pipeline of ingest modules. + * Executes an ingest task by passing the task to the appropriate ingest + * task pipeline. * * @param task The ingest task. */ void execute(IngestTask task) { /* - * The following "switch on actual type" eliminates code duplication in - * the IngestTask hierarchy. Future work may or may not be able to - * eliminate the switch. + * NOTE: The following switch on task type enables elimination of code + * duplication in the IngestTask hierarchy. Future work may or may not + * be able to eliminate this switch. */ if (task instanceof DataSourceIngestTask) { executeDataSourceIngestTask((DataSourceIngestTask) task); @@ -995,83 +1127,88 @@ final class IngestJobPipeline { } /** - * Passes a data source for this job through the currently active data - * source level ingest pipeline. + * Passes the data source for the ingest job through the currently active + * data source level ingest task pipeline (first stage or second stage data + * source ingest modules). * * @param task A data source ingest task wrapping the data source. */ private void executeDataSourceIngestTask(DataSourceIngestTask task) { try { - synchronized (this.dataSourceIngestPipelineLock) { - if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { + synchronized (dataSourceIngestPipelineLock) { + if (!isCancelled() && !currentDataSourceIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.performTask(task)); + errors.addAll(currentDataSourceIngestPipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } } } - if (this.doUI) { + if (doUI) { /** * Shut down the data source ingest progress bar right away. * Data source-level processing is finished for this stage. */ - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgress != null) { + dataSourceIngestProgress.finish(); + dataSourceIngestProgress = null; } } } } finally { - IngestJobPipeline.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); + taskScheduler.notifyTaskCompleted(task); + checkForStageCompleted(); } } /** - * Passes a file from the data source for this job through the file ingest - * pipeline. + * Passes a file from the data source for the ingest job through the file + * ingest task pipeline (file ingest modules). * * @param task A file ingest task wrapping the file. */ private void executeFileIngestTask(FileIngestTask task) { try { - if (!this.isCancelled()) { - FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take(); + if (!isCancelled()) { + FileIngestPipeline pipeline = fileIngestPipelinesQueue.take(); if (!pipeline.isEmpty()) { + /* + * Get the file from the task. If the file was "streamed," + * the task may only have the file object ID and a trip to + * the case database will be required. + */ AbstractFile file; try { file = task.getFile(); } catch (TskCoreException ex) { - // In practice, this task would never have been enqueued since the file - // lookup would have failed there. List errors = new ArrayList<>(); - errors.add(new IngestModuleError("Ingest Job Pipeline", ex)); + errors.add(new IngestModuleError("Ingest Pipeline", ex)); logIngestModuleErrors(errors); - this.fileIngestPipelinesQueue.put(pipeline); + fileIngestPipelinesQueue.put(pipeline); return; } - synchronized (this.fileIngestProgressLock) { - ++this.processedFiles; - if (this.doUI) { + synchronized (fileIngestProgressLock) { + ++processedFiles; + if (doUI) { /** - * Update the file ingest progress bar. + * Update the file ingest progress bar in the lower + * right hand corner of the main application window. */ - if (this.processedFiles <= this.estimatedFilesToProcess) { - this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); + if (processedFiles <= estimatedFilesToProcess) { + fileIngestProgress.progress(file.getName(), (int) processedFiles); } else { - this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); + fileIngestProgress.progress(file.getName(), (int) estimatedFilesToProcess); } - this.filesInProgress.add(file.getName()); + filesInProgress.add(file.getName()); // RJCTODO: Snapshot data structure only updated for UI case, I think this is wrong } } /** - * Run the file through the pipeline. + * Run the file through the modules in the pipeline. */ List errors = new ArrayList<>(); errors.addAll(pipeline.performTask(task)); @@ -1079,101 +1216,127 @@ final class IngestJobPipeline { logIngestModuleErrors(errors, file); } - if (this.doUI && !this.cancelled) { - synchronized (this.fileIngestProgressLock) { + if (doUI && !cancelled) { + synchronized (fileIngestProgressLock) { /** * Update the file ingest progress bar again, in * case the file was being displayed. */ - this.filesInProgress.remove(file.getName()); - if (this.filesInProgress.size() > 0) { - this.fileIngestProgress.progress(this.filesInProgress.get(0)); + filesInProgress.remove(file.getName()); // RJCTODO: Snapshot data structure only updated for UI case, I think this is wrong + if (filesInProgress.size() > 0) { + fileIngestProgress.progress(filesInProgress.get(0)); } else { - this.fileIngestProgress.progress(""); + fileIngestProgress.progress(""); } } } } - this.fileIngestPipelinesQueue.put(pipeline); + fileIngestPipelinesQueue.put(pipeline); } } catch (InterruptedException ex) { - // RJCTODO + // RJCTODO This probablly should be logged, interrupt during wait for pipeline copy } finally { - IngestJobPipeline.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); + taskScheduler.notifyTaskCompleted(task); + checkForStageCompleted(); } } /** - * Passes a data artifact from the data source for this job through the data - * artifact ingest pipeline. + * Passes a data artifact from the data source for the ingest job through + * the data artifact ingest task pipeline (data artifact ingest modules). * * @param task A data artifact ingest task wrapping the file. */ private void executeDataArtifactIngestTask(DataArtifactIngestTask task) { - // RJCTODO + try { + if (!isCancelled() && !artifactIngestPipeline.isEmpty()) { + List errors = new ArrayList<>(); + errors.addAll(artifactIngestPipeline.performTask(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + } finally { + taskScheduler.notifyTaskCompleted(task); // RJCTODO: Looks like another overload is needed or a switch case + checkForStageCompleted(); + } } /** - * Add a list of files (by object ID) to the ingest queue. Must call start() - * prior to adding files. + * Adds some subset of the "streamed" files for a stremaing ingest job to + * this pipeline after startUp() has been called. * - * @param fileObjIds List of newly added file IDs. + * @param fileObjIds The object IDs of the files. */ void addStreamingIngestFiles(List fileObjIds) { - - // Return if there are no file ingest modules enabled. - if (!hasFileIngestPipeline()) { - return; - } - - if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY)) { - IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + if (hasFileIngestModules()) { + if (stage.equals(Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY)) { + IngestPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + } } } /** - * Starts data source ingest. Should be called after the data source - * processor has finished (i.e., all files are in the database) + * Adds the data source for a streaming ingest job to this pipeline after + * startUp() has been called. Intended to be called after the data source + * processor has finished its processing (i.e., all the file system files + * for the data source are in the case database). */ - void processStreamingIngestDataSource() { - startDataSourceIngestStreaming(); + void addStreamingIngestDataSource() { + startDataSourceTaskInStreamingMode(); checkForStageCompleted(); } /** - * Adds more files from the data source for this job to the job, e.g., adds - * extracted or carved files. Not currently supported for the second stage - * of the job. + * Adds additional files (e.g., extracted or carved files) for any type of + * ingest job to this pipeline after startUp() has been called. Not + * currently supported for second stage of the job. * * @param files A list of the files to add. */ void addFiles(List files) { - if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY) - || stage.equals(Stages.FIRST_STAGE_FILES_AND_DATASOURCE)) { - IngestJobPipeline.taskScheduler.fastTrackFileIngestTasks(this, files); + if (stage.equals(Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY) + || stage.equals(Stages.FIRST_STAGE_ALL_TASKS)) { + taskScheduler.fastTrackFileIngestTasks(this, files); } else { - logErrorMessage(Level.SEVERE, "Adding files to job during second stage analysis not supported"); + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } /** * The intended clients of this method are ingest modules running code - * on an ingest thread that is holding a reference to an ingest task, in - * which case a completion check would not be necessary, so this is a - * bit of defensive programming. + * in an ingest thread that is holding a reference to a "primary" ingest + * task that was the source of the files, in which case a completion + * check would not be necessary, so this is a bit of defensive + * programming. */ - this.checkForStageCompleted(); + checkForStageCompleted(); } /** - * RJCTODO + * Adds data artifacts for any type of ingest job to this pipeline after + * startUp() has been called. * * @param artifacts */ void addDataArtifacts(List artifacts) { - // RJCTODO + if (stage.equals(Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY) + || stage.equals(Stages.FIRST_STAGE_ALL_TASKS) + || stage.equals(Stages.SECOND_STAGE)) { + //taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); // RJCTODO + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + } + + /** + * The intended clients of this method are ingest modules running code + * in an ingest thread that is holding a reference to a "primary" ingest + * task that was the source of the files, in which case a completion + * check would not be necessary, so this is a bit of defensive + * programming. + */ + checkForStageCompleted(); } /** @@ -1229,6 +1392,7 @@ final class IngestJobPipeline { * * @param workUnits Number of work units performed. */ + // RJCTODO: Is this used? void advanceDataSourceIngestProgressBar(int workUnits) { if (this.doUI && !this.cancelled) { synchronized (this.dataSourceIngestProgressLock) { @@ -1338,7 +1502,7 @@ final class IngestJobPipeline { void cancel(IngestJob.CancellationReason reason) { this.cancelled = true; this.cancellationReason = reason; - IngestJobPipeline.taskScheduler.cancelPendingTasksForIngestJob(this); + IngestPipeline.taskScheduler.cancelPendingTasksForIngestJob(this); if (this.doUI) { synchronized (this.dataSourceIngestProgressLock) { @@ -1368,7 +1532,7 @@ final class IngestJobPipeline { * @param taskName Name of file the module is running on. */ void setCurrentFileIngestModule(String moduleName, String taskName) { - currentFileIngestTask = taskName; + currentFileIngestTask = taskName; //RJCTODO: Is this correct? } /** @@ -1397,7 +1561,7 @@ final class IngestJobPipeline { * @param message The message. */ private void logInfoMessage(String message) { - logger.log(Level.INFO, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJob.getIngestJobId())); //NON-NLS + logger.log(Level.INFO, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJobInfo.getIngestJobId())); //NON-NLS } /** @@ -1409,7 +1573,7 @@ final class IngestJobPipeline { * @param throwable The throwable associated with the error. */ private void logErrorMessage(Level level, String message, Throwable throwable) { - logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJob.getIngestJobId()), throwable); //NON-NLS + logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJobInfo.getIngestJobId()), throwable); //NON-NLS } /** @@ -1420,7 +1584,7 @@ final class IngestJobPipeline { * @param message The message. */ private void logErrorMessage(Level level, String message) { - logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJob.getIngestJobId())); //NON-NLS + logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJobInfo.getIngestJobId())); //NON-NLS } /** @@ -1447,7 +1611,7 @@ final class IngestJobPipeline { } /** - * Gets a snapshot of this jobs state and performance. + * Gets a snapshot of this ingest pipelines current state. * * @return An ingest job statistics object. */ @@ -1459,7 +1623,6 @@ final class IngestJobPipeline { */ boolean fileIngestRunning = false; Date fileIngestStartTime = null; - for (FileIngestPipeline pipeline : this.fileIngestPipelines) { if (pipeline.isRunning()) { fileIngestRunning = true; @@ -1470,23 +1633,27 @@ final class IngestJobPipeline { } } + /* + * RJCTODO + */ long processedFilesCount = 0; long estimatedFilesToProcessCount = 0; long snapShotTime = new Date().getTime(); IngestJobTasksSnapshot tasksSnapshot = null; - if (getIngestTasksSnapshot) { synchronized (fileIngestProgressLock) { processedFilesCount = this.processedFiles; estimatedFilesToProcessCount = this.estimatedFilesToProcess; snapShotTime = new Date().getTime(); } - tasksSnapshot = IngestJobPipeline.taskScheduler.getTasksSnapshotForJob(pipelineId); + tasksSnapshot = IngestPipeline.taskScheduler.getTasksSnapshotForJob(pipelineId); } - return new Snapshot(this.dataSource.getName(), pipelineId, createTime, - getCurrentDataSourceIngestModule(), fileIngestRunning, fileIngestStartTime, + return new Snapshot(dataSource.getName(), + pipelineId, createTime, + getCurrentDataSourceIngestModule(), + fileIngestRunning, fileIngestStartTime, cancelled, cancellationReason, cancelledDataSourceIngestModules, processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index 23a323690a..dff29f9369 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -29,7 +29,7 @@ import org.sleuthkit.datamodel.Content; abstract class IngestTask { private final static long NOT_SET = Long.MIN_VALUE; - private final IngestJobPipeline ingestJobPipeline; + private final IngestPipeline ingestJobPipeline; private long threadId; /** @@ -41,7 +41,7 @@ abstract class IngestTask { * @param ingestJobPipeline The ingest job pipeline to use to execute the * task. */ - IngestTask(IngestJobPipeline ingestJobPipeline) { + IngestTask(IngestPipeline ingestJobPipeline) { this.ingestJobPipeline = ingestJobPipeline; threadId = NOT_SET; } @@ -51,7 +51,7 @@ abstract class IngestTask { * * @return The ingest job pipeline. */ - IngestJobPipeline getIngestJobPipeline() { + IngestPipeline getIngestJobPipeline() { return ingestJobPipeline; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java index f7451dd9d8..1662d890df 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java @@ -37,7 +37,7 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; abstract class IngestTaskPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestJobPipeline ingestJobPipeline; + private final IngestPipeline ingestJobPipeline; private final List moduleTemplates; private final List> modules; private volatile Date startTime; @@ -56,7 +56,7 @@ abstract class IngestTaskPipeline { * @param moduleTemplates The ingest module templates that define this * pipeline. */ - IngestTaskPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + IngestTaskPipeline(IngestPipeline ingestJobPipeline, List moduleTemplates) { this.ingestJobPipeline = ingestJobPipeline; this.moduleTemplates = moduleTemplates; modules = new ArrayList<>(); @@ -322,7 +322,7 @@ abstract class IngestTaskPipeline { * @throws IngestModuleException Excepton thrown if there is an error * performing the task. */ - abstract void performTask(IngestJobPipeline ingestJobPipeline, T task) throws IngestModuleException; + abstract void performTask(IngestPipeline ingestJobPipeline, T task) throws IngestModuleException; @Override public void shutDown() { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 7eedcf12aa..72efbcd005 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -36,8 +36,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataArtifact; import org.sleuthkit.datamodel.FileSystem; @@ -127,7 +129,7 @@ final class IngestTasksScheduler { * * @param ingestJobPipeline The ingest job pipeline. */ - synchronized void scheduleIngestTasks(IngestJobPipeline ingestJobPipeline) { + synchronized void scheduleDataSourceAndFileIngestTasks(IngestPipeline ingestJobPipeline) { if (!ingestJobPipeline.isCancelled()) { /* * Scheduling of both the data source ingest task and the initial @@ -147,7 +149,7 @@ final class IngestTasksScheduler { * * @param ingestJobPipeline The ingest job pipeline. */ - synchronized void scheduleDataSourceIngestTask(IngestJobPipeline ingestJobPipeline) { + synchronized void scheduleDataSourceIngestTask(IngestPipeline ingestJobPipeline) { if (!ingestJobPipeline.isCancelled()) { DataSourceIngestTask task = new DataSourceIngestTask(ingestJobPipeline); try { @@ -168,7 +170,7 @@ final class IngestTasksScheduler { * empty, then file tasks for all files in the data * source are scheduled. */ - synchronized void scheduleFileIngestTasks(IngestJobPipeline ingestJobPipeline, Collection files) { + synchronized void scheduleFileIngestTasks(IngestPipeline ingestJobPipeline, Collection files) { if (!ingestJobPipeline.isCancelled()) { Collection candidateFiles; if (files.isEmpty()) { @@ -194,7 +196,7 @@ final class IngestTasksScheduler { * empty, then file tasks for all files in the data * source are scheduled. */ - synchronized void scheduleStreamedFileIngestTasks(IngestJobPipeline ingestJobPipeline, List fileIds) { + synchronized void scheduleStreamedFileIngestTasks(IngestPipeline ingestJobPipeline, List fileIds) { if (!ingestJobPipeline.isCancelled()) { for (long id : fileIds) { // Create the file ingest task. Note that we do not do the shouldEnqueueFileTask() @@ -214,7 +216,7 @@ final class IngestTasksScheduler { * @param ingestJobPipeline The ingestJobPipeline. * @param files A set of files for the data source. */ - synchronized void fastTrackFileIngestTasks(IngestJobPipeline ingestJobPipeline, Collection files) { + synchronized void fastTrackFileIngestTasks(IngestPipeline ingestJobPipeline, Collection files) { if (!ingestJobPipeline.isCancelled()) { /* * Put the files directly into the queue for the file ingest @@ -240,19 +242,39 @@ final class IngestTasksScheduler { } /** - * Schedules a data artifact ingest task for an ingest job pipeline. + * Schedules data artifact ingest tasks for a given ingest pipeline. * - * @param ingestJobPipeline The ingest job pipeline. + * @param ingestPipeline The ingest pipeline. */ - synchronized void scheduleDataArtifactIngestTasks(IngestJobPipeline ingestJobPipeline, Collection artifacts) { - if (!ingestJobPipeline.isCancelled()) { + synchronized void scheduleDataArtifactIngestTasks(IngestPipeline ingestPipeline) { + if (!ingestPipeline.isCancelled()) { + Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); + try { + List artifacts = blackboard.getDataArtifacts(ingestPipeline.getDataSource()); + scheduleDataArtifactIngestTasks(ingestPipeline, artifacts); + } catch (TskCoreException ex) { + // RJCTODO + } + } + } + + /** + * RJCTODO + * + * @param ingestPipeline + * @param artifacts + */ + synchronized void scheduleDataArtifactIngestTasks(IngestPipeline ingestPipeline, List artifacts) { // RJCTODO: WHy are cancellation checks in the scheduler instead of in the pipeline? + if (!ingestPipeline.isCancelled()) { for (DataArtifact artifact : artifacts) { - DataArtifactIngestTask task = new DataArtifactIngestTask(ingestJobPipeline, artifact); + DataArtifactIngestTask task = new DataArtifactIngestTask(ingestPipeline, artifact); try { this.dataArtifactIngestThreadQueue.putLast(task); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data artifact ingest task queue (jobId={%d)", ingestJobPipeline.getId()), ex); + // RJCTODO: Better logging + IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data artifact ingest task queue (jobId={%d)", ingestPipeline.getId()), ex); Thread.currentThread().interrupt(); + break; } } } @@ -298,7 +320,7 @@ final class IngestTasksScheduler { * * @return True or false. */ - synchronized boolean currentTasksAreCompleted(IngestJobPipeline ingestJobPipeline) { + synchronized boolean currentTasksAreCompleted(IngestPipeline ingestJobPipeline) { long jobId = ingestJobPipeline.getId(); return !(this.dataSourceIngestThreadQueue.hasTasksForJob(jobId) @@ -316,7 +338,7 @@ final class IngestTasksScheduler { * * @param ingestJobPipeline The ingestJobPipeline. */ - synchronized void cancelPendingTasksForIngestJob(IngestJobPipeline ingestJobPipeline) { + synchronized void cancelPendingTasksForIngestJob(IngestPipeline ingestJobPipeline) { long jobId = ingestJobPipeline.getId(); IngestTasksScheduler.removeTasksForJob(rootFileTaskQueue, jobId); IngestTasksScheduler.removeTasksForJob(pendingFileTaskQueue, jobId); @@ -622,7 +644,7 @@ final class IngestTasksScheduler { return false; } } - + /** * Checks whether or not a collection of ingest tasks includes a task for a * given data source ingest job. @@ -692,6 +714,7 @@ final class IngestTasksScheduler { this.dataSourceIngestThreadQueue.countRunningTasksForJob(jobId) + this.fileIngestThreadsQueue.countRunningTasksForJob(jobId), countTasksForJob(this.streamedTasksQueue, jobId)); // RJCTODO + } /** From 57fdd0764e6eba8be04720c3681b3ba91d23f6b2 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 9 Mar 2021 15:02:24 -0500 Subject: [PATCH 05/63] 7332 Artifacts ingest pipeline, interim commit --- .../CentralRepoIngestModuleFactory.java | 6 +- .../autopsy/ingest/FileIngestTask.java | 6 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 1 + .../autopsy/ingest/IngestJobContext.java | 164 +++-- .../autopsy/ingest/IngestManager.java | 6 +- .../autopsy/ingest/IngestPipeline.java | 2 +- .../sleuthkit/autopsy/ingest/IngestTask.java | 4 +- .../autopsy/ingest/IngestTasksScheduler.java | 599 +++++++++++------- 8 files changed, 466 insertions(+), 322 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java index b6ab3ae52d..7fbf162e83 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java @@ -34,8 +34,10 @@ import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; * An ingest module factory for Central Repository ingest modules. */ @ServiceProvider(service = org.sleuthkit.autopsy.ingest.IngestModuleFactory.class) -@NbBundle.Messages({"CentralRepoIngestModuleFactory.ingestmodule.name=Central Repository", - "CentralRepoIngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation"}) +@NbBundle.Messages({ + "CentralRepoIngestModuleFactory.ingestmodule.name=Central Repository", + "CentralRepoIngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation" +}) public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 1b17c27f98..76769e893e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -95,8 +95,8 @@ final class FileIngestTask extends IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - IngestPipeline thisPipeline = getIngestJobPipeline(); - IngestPipeline otherPipeline = other.getIngestJobPipeline(); + IngestPipeline thisPipeline = getIngestPipeline(); + IngestPipeline otherPipeline = other.getIngestPipeline(); if (thisPipeline != otherPipeline && (thisPipeline == null || !thisPipeline.equals(otherPipeline))) { return false; } @@ -106,7 +106,7 @@ final class FileIngestTask extends IngestTask { @Override public int hashCode() { int hash = 5; - hash = 47 * hash + Objects.hashCode(getIngestJobPipeline()); + hash = 47 * hash + Objects.hashCode(getIngestPipeline()); hash = 47 * hash + Objects.hashCode(this.fileId); return hash; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 451a9b06cc..20b8b65592 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -136,6 +136,7 @@ public final class IngestJob { * @return The job identifier. */ public long getId() { + // RJCTODO: Replace this with the pipeline ID, as has been done elsewhere. return this.id; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 2c3f1e556e..65b660dc8e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,44 +21,125 @@ package org.sleuthkit.autopsy.ingest; import java.util.List; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataArtifact; /** - * Provides an ingest module with services specific to the ingest job of which - * the module is a part. + * An ingest job context object that provides an ingest module with operations + * specific to the ingest job of which the module is a part. + * + * From a coding perspective, this object provides an abstraction that hides + * ingest infrastructure details. */ public final class IngestJobContext { - private final IngestPipeline ingestJobPipeline; + private final IngestPipeline ingestPipeline; + /** + * Constructs an ingest job context object that provides an ingest module + * with operations specific to the ingest job of which the module is a part. + * + * @param ingestJobPipeline + */ IngestJobContext(IngestPipeline ingestJobPipeline) { - this.ingestJobPipeline = ingestJobPipeline; + this.ingestPipeline = ingestJobPipeline; } /** - * Gets the ingest job execution context identifier. + * Gets the execution context identifier of the ingest job. * * @return The context string. */ public String getExecutionContext() { - return this.ingestJobPipeline.getExecutionContext(); + return this.ingestPipeline.getExecutionContext(); } - + /** - * Gets the data source associated with this context. + * Gets the data source for the ingest job. * * @return The data source. */ public Content getDataSource() { - return this.ingestJobPipeline.getDataSource(); + return ingestPipeline.getDataSource(); } /** - * Gets the identifier of the ingest job associated with this context. + * Gets the unique identifier for the ingest job. * - * @return The ingest job identifier. + * @return The ID. */ public long getJobId() { - return this.ingestJobPipeline.getId(); + return ingestPipeline.getId(); + } + + /** + * Checks whether or not cancellation of the currently running data source + * level ingest module for the ingest job has been requested. Data source + * level ingest modules should check this periodically and break off + * processing if the method returns true. + * + * @return True or false. + */ + public boolean dataSourceIngestIsCancelled() { + return ingestPipeline.currentDataSourceIngestModuleIsCancelled() || ingestPipeline.isCancelled(); + } + + /** + * Checks whether or not cancellation of the currently running file ingest + * module for the ingest job has been requested. File ingest modules should + * check this periodically and break off processing if the method returns + * true. + * + * RJCTODO: Figure out how to check/handle cancellation for data artoifact + * ingest modules. Perhaps it is time to add a cancel method with an + * implementation in teh "adapter" classes. + * + * @return True or false. + */ + public boolean fileIngestIsCancelled() { + return ingestPipeline.isCancelled(); + } + + /** + * Queries whether or not unallocated space should be processed for the + * ingest job. + * + * @return True or false. + */ + public boolean processingUnallocatedSpace() { + return ingestPipeline.shouldProcessUnallocatedSpace(); + } + + /** + * Adds one or more files, e.g., extracted or carved files, to the ingest + * job for processing by its file ingest modules. + * + * @param files The files. + */ + public void addFilesToJob(List files) { + ingestPipeline.addFiles(files); + } + + /** + * Adds one or more data artifacts to the ingest job for processing by its + * data artifact ingest modules. + * + * @param artifacts The artifacts. + */ + public void addDataArtifactsToJob(List artifacts) { + ingestPipeline.addDataArtifacts(artifacts); + } + + /** + * Adds one or more files, i.e., extracted or carved files, to the ingest + * job associated with this context. + * + * @param files The files to be added. + * + * @deprecated use addFilesToJob() instead. + */ + @Deprecated + public void scheduleFiles(List files) { + addFilesToJob(files); } /** @@ -72,62 +153,7 @@ public final class IngestJobContext { */ @Deprecated public boolean isJobCancelled() { - return this.dataSourceIngestIsCancelled(); - } - - /** - * Allows a data source ingest module to determine whether or not - * cancellation of the data source ingest part of the ingest job associated - * with this context has been requested. - * - * @return True or false. - */ - public boolean dataSourceIngestIsCancelled() { - return this.ingestJobPipeline.currentDataSourceIngestModuleIsCancelled() || this.ingestJobPipeline.isCancelled(); - } - - /** - * Allows a file ingest module to determine whether or not cancellation of - * the file ingest part of the ingest job associated with this context has - * been requested. - * - * @return True or false. - */ - public boolean fileIngestIsCancelled() { - return this.ingestJobPipeline.isCancelled(); - } - - /** - * Queries whether or not unallocated space should be processed for the - * ingest job associated with this context. - * - * @return True or false. - */ - public boolean processingUnallocatedSpace() { - return this.ingestJobPipeline.shouldProcessUnallocatedSpace(); - } - - /** - * Adds one or more files, i.e., extracted or carved files, to the ingest - * job associated with this context. - * - * @param files The files to be added. - * - * @deprecated use addFilesToJob() instead - */ - @Deprecated - public void scheduleFiles(List files) { - this.addFilesToJob(files); - } - - /** - * Adds one or more files, i.e., extracted or carved files, to the ingest - * job associated with this context. - * - * @param files The files to be added. - */ - public void addFilesToJob(List files) { - this.ingestJobPipeline.addFiles(files); + return dataSourceIngestIsCancelled(); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 0a0fb12f07..210392e16f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -767,7 +767,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { * the task. */ void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) { - ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJobPipeline().getId(), ingestModuleDisplayName, task.getDataSource())); + ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestPipeline().getId(), ingestModuleDisplayName, task.getDataSource())); } /** @@ -785,11 +785,11 @@ public class IngestManager implements IngestProgressSnapshotProvider { IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); IngestThreadActivitySnapshot newSnap; try { - newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJobPipeline().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile()); + newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestPipeline().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile()); } catch (TskCoreException ex) { // In practice, this task would never have been enqueued or processed since the file // lookup would have failed. - newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJobPipeline().getId(), ingestModuleDisplayName, task.getDataSource()); + newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestPipeline().getId(), ingestModuleDisplayName, task.getDataSource()); } ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java index e3d04eb4bb..e76555d9e4 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java @@ -1502,7 +1502,7 @@ final class IngestPipeline { void cancel(IngestJob.CancellationReason reason) { this.cancelled = true; this.cancellationReason = reason; - IngestPipeline.taskScheduler.cancelPendingTasksForIngestJob(this); + IngestPipeline.taskScheduler.cancelPendingFileTasksForIngestJob(this); if (this.doUI) { synchronized (this.dataSourceIngestProgressLock) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index dff29f9369..fde3449b75 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -51,7 +51,7 @@ abstract class IngestTask { * * @return The ingest job pipeline. */ - IngestPipeline getIngestJobPipeline() { + IngestPipeline getIngestPipeline() { return ingestJobPipeline; } @@ -61,7 +61,7 @@ abstract class IngestTask { * @return The data source. */ Content getDataSource() { - return getIngestJobPipeline().getDataSource(); + return getIngestPipeline().getDataSource(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 72efbcd005..090b13a053 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -42,13 +42,14 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataArtifact; +import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.FileSystem; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; /** - * Creates ingest tasks for data source ingest jobs, queueing the tasks in - * priority order for execution by the ingest manager's ingest threads. + * Creates ingest tasks for ingest jobs, queueing the tasks in priority order + * for execution by the ingest manager's ingest threads. */ @ThreadSafe final class IngestTasksScheduler { @@ -57,20 +58,20 @@ final class IngestTasksScheduler { private static final Logger logger = Logger.getLogger(IngestTasksScheduler.class.getName()); @GuardedBy("IngestTasksScheduler.this") private static IngestTasksScheduler instance; - private final IngestTaskTrackingQueue dataSourceIngestThreadQueue; + private final IngestTaskTrackingQueue dataSourceIngestTasksQueue; @GuardedBy("this") - private final TreeSet rootFileTaskQueue; + private final TreeSet topLevelFileIngestTasksQueue; @GuardedBy("this") - private final Deque pendingFileTaskQueue; + private final Deque batchedFileIngestTasksQueue; @GuardedBy("this") - private final Queue streamedTasksQueue; - private final IngestTaskTrackingQueue fileIngestThreadsQueue; - private final IngestTaskTrackingQueue dataArtifactIngestThreadQueue; + private final Queue streamedFileIngestTasksQueue; + private final IngestTaskTrackingQueue fileIngestTasksQueue; + private final IngestTaskTrackingQueue artifactIngestTasksQueue; /** * Gets the ingest tasks scheduler singleton that creates ingest tasks for - * data source ingest jobs, queueing the tasks in priority order for - * execution by the ingest manager's ingest threads. + * ingest jobs, queueing the tasks in priority order for execution by the + * ingest manager's ingest threads. */ synchronized static IngestTasksScheduler getInstance() { if (IngestTasksScheduler.instance == null) { @@ -80,159 +81,246 @@ final class IngestTasksScheduler { } /** - * Constructs an ingest tasks scheduler that creates ingest tasks for data - * source ingest jobs, queueing the tasks in priority order for execution by - * the ingest manager's ingest threads. + * Constructs an ingest tasks scheduler that creates ingest tasks for ingest + * jobs, queueing the tasks in priority order for execution by the ingest + * manager's ingest threads. */ private IngestTasksScheduler() { - dataSourceIngestThreadQueue = new IngestTaskTrackingQueue(); - rootFileTaskQueue = new TreeSet<>(new RootDirectoryTaskComparator()); - pendingFileTaskQueue = new LinkedList<>(); - fileIngestThreadsQueue = new IngestTaskTrackingQueue(); - streamedTasksQueue = new LinkedList<>(); - dataArtifactIngestThreadQueue = new IngestTaskTrackingQueue(); + dataSourceIngestTasksQueue = new IngestTaskTrackingQueue(); + topLevelFileIngestTasksQueue = new TreeSet<>(new RootDirectoryTaskComparator()); + batchedFileIngestTasksQueue = new LinkedList<>(); + fileIngestTasksQueue = new IngestTaskTrackingQueue(); + streamedFileIngestTasksQueue = new LinkedList<>(); + artifactIngestTasksQueue = new IngestTaskTrackingQueue(); } /** * Gets the data source level ingest tasks queue. This queue is a blocking - * queue used by the ingest manager's data source level ingest thread. + * queue consumed by the ingest manager's data source level ingest thread. * * @return The queue. */ BlockingIngestTaskQueue getDataSourceIngestTaskQueue() { - return dataSourceIngestThreadQueue; + return dataSourceIngestTasksQueue; } /** * Gets the file level ingest tasks queue. This queue is a blocking queue - * used by the ingest manager's file level ingest threads. + * consumed by the ingest manager's file level ingest threads. * * @return The queue. */ BlockingIngestTaskQueue getFileIngestTaskQueue() { - return fileIngestThreadsQueue; + return fileIngestTasksQueue; } /** - * Gets the data artifacts ingest tasks queue. This queue is a blocking - * queue used by the ingest manager's data artifact ingest thread. + * Gets the data artifact ingest tasks queue. This queue is a blocking queue + * consumed by the ingest manager's data artifact ingest thread. * * @return The queue. */ BlockingIngestTaskQueue getDataArtifactIngestTaskQueue() { - return dataArtifactIngestThreadQueue; + return artifactIngestTasksQueue; } /** - * Schedules a data source level ingest task and zero to many file level - * ingest tasks for an ingest job pipeline. + * Schedules a data source level ingest task and the initial file and result + * ingest tasks for an ingest job. Note that the file filter for the job is + * obtained from the ingest pipeline for the job and that applying the + * filter may cause some or even all of the files in the data source to not + * be scheduled. * - * @param ingestJobPipeline The ingest job pipeline. + * Note that task scheduling for an ingest job and checking if all of the + * job's tasks are completed must always be atomic operations, enforced by + * synchronization using the scheduler's monitor. Otherwise, the completion + * checks could result in false positives. + * + * There is also a necessary convention that child tasks for products of + * other tasks in the job need to be scheduled before the parent tasks + * notify the scheduler that they are completed. This avoids false positives + * for completion checks by ensuring that the child tasks are queued before + * the parent tasks are marked as completed. + * + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. */ - synchronized void scheduleDataSourceAndFileIngestTasks(IngestPipeline ingestJobPipeline) { - if (!ingestJobPipeline.isCancelled()) { - /* - * Scheduling of both the data source ingest task and the initial - * file ingest tasks for an ingestJobPipeline must be an atomic - * operation. Otherwise, the data source task might be completed - * before the file tasks are scheduled, resulting in a potential - * false positive when another thread checks whether or not all the - * tasks for the ingestJobPipeline are completed. - */ - this.scheduleDataSourceIngestTask(ingestJobPipeline); - this.scheduleFileIngestTasks(ingestJobPipeline, Collections.emptyList()); + synchronized void scheduleDataSourceAndFileIngestTasks(IngestPipeline ingestPipeline) { + if (!ingestPipeline.isCancelled()) { + scheduleDataSourceIngestTask(ingestPipeline); + scheduleFileIngestTasks(ingestPipeline, Collections.emptyList()); + scheduleDataArtifactIngestTasks(ingestPipeline); } } /** - * Schedules a data source level ingest task for an ingest job pipeline. + * Schedules a data source level ingest task for an ingest job. * - * @param ingestJobPipeline The ingest job pipeline. + * Note that task scheduling for an ingest job and checking if all of the + * job's tasks are completed must always be atomic operations, enforced by + * synchronization using the scheduler's monitor. Otherwise, the completion + * checks could result in false positives. + * + * There is also a necessary convention that child tasks for products of + * other tasks in the job need to be scheduled before the parent tasks + * notify the scheduler that they are completed. This avoids false positives + * for completion checks by ensuring that the child tasks are queued before + * the parent tasks are marked as completed. + * + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. */ - synchronized void scheduleDataSourceIngestTask(IngestPipeline ingestJobPipeline) { - if (!ingestJobPipeline.isCancelled()) { - DataSourceIngestTask task = new DataSourceIngestTask(ingestJobPipeline); + synchronized void scheduleDataSourceIngestTask(IngestPipeline ingestPipeline) { + if (!ingestPipeline.isCancelled()) { + DataSourceIngestTask task = new DataSourceIngestTask(ingestPipeline); try { - this.dataSourceIngestThreadQueue.putLast(task); + dataSourceIngestTasksQueue.putLast(task); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data source level ingest task queue (jobId={%d)", ingestJobPipeline.getId()), ex); + IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data source level ingest task queue (pipelineId={%d)", ingestPipeline.getId()), ex); Thread.currentThread().interrupt(); } } } /** - * Schedules file tasks for either all the files or a given subset of the - * files for an ingest job pipeline. + * Schedules file tasks for either all the files, or a given subset of the + * files, for an ingest job. Note that the file filter for the job is + * obtained from the ingest pipeline and its application may cause some or + * even all of the files to not be scheduled. * - * @param ingestJobPipeline The ingest job pipeline. - * @param files A subset of the files for the data source; if - * empty, then file tasks for all files in the data - * source are scheduled. + * Note that task scheduling for an ingest job and checking if all of the + * job's tasks are completed must always be atomic operations, enforced by + * synchronization using the scheduler's monitor. Otherwise, the completion + * checks could result in false positives. + * + * There is also a necessary convention that child tasks for products of + * other tasks in the job need to be scheduled before the parent tasks + * notify the scheduler that they are completed. This avoids false positives + * for completion checks by ensuring that the child tasks are queued before + * the parent tasks are marked as completed. + * + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. + * @param files A subset of the files from the data source; if + * empty, then all if the files from the data source + * are candidates for scheduling. */ - synchronized void scheduleFileIngestTasks(IngestPipeline ingestJobPipeline, Collection files) { - if (!ingestJobPipeline.isCancelled()) { + synchronized void scheduleFileIngestTasks(IngestPipeline ingestPipeline, Collection files) { + if (!ingestPipeline.isCancelled()) { Collection candidateFiles; if (files.isEmpty()) { - candidateFiles = getTopLevelFiles(ingestJobPipeline.getDataSource()); + candidateFiles = getTopLevelFiles(ingestPipeline.getDataSource()); } else { candidateFiles = files; } for (AbstractFile file : candidateFiles) { - FileIngestTask task = new FileIngestTask(ingestJobPipeline, file); + FileIngestTask task = new FileIngestTask(ingestPipeline, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { - this.rootFileTaskQueue.add(task); + topLevelFileIngestTasksQueue.add(task); } } - refillIngestThreadQueue(); + refillFileIngestTasksQueue(); } } /** - * Schedules file tasks for the given list of file IDs. + * Schedules file tasks for a collection of "streamed" files for a streaming + * ingest job. Note that the file filter for the job is obtained from the + * ingest pipeline and its application may cause some or even all of the + * files to not be scheduled. * - * @param ingestJobPipeline The ingest job pipeline. - * @param files A subset of the files for the data source; if - * empty, then file tasks for all files in the data - * source are scheduled. + * Note that task scheduling for an ingest job and checking if all of the + * job's tasks are completed must always be atomic operations, enforced by + * synchronization using the scheduler's monitor. Otherwise, the completion + * checks could result in false positives. + * + * There is also a necessary convention that child tasks for products of + * other tasks in the job need to be scheduled before the parent tasks + * notify the scheduler that they are completed. This avoids false positives + * for completion checks by ensuring that the child tasks are queued before + * the parent tasks are marked as completed. + * + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. + * @param files A list of file object IDs for the streamed files. */ - synchronized void scheduleStreamedFileIngestTasks(IngestPipeline ingestJobPipeline, List fileIds) { - if (!ingestJobPipeline.isCancelled()) { + synchronized void scheduleStreamedFileIngestTasks(IngestPipeline ingestPipeline, List fileIds) { + if (!ingestPipeline.isCancelled()) { for (long id : fileIds) { - // Create the file ingest task. Note that we do not do the shouldEnqueueFileTask() - // check here in order to delay loading the AbstractFile object. - FileIngestTask task = new FileIngestTask(ingestJobPipeline, id); - this.streamedTasksQueue.add(task); + /* + * Create the file ingest task. Note that we do not do the + * shouldEnqueueFileTask() check here in order to delay querying + * the case database to construct the AbstractFile object. The + * file filter will be applied before the file task makes it to + * the task queue consumed by the file ingest threads. + */ + FileIngestTask task = new FileIngestTask(ingestPipeline, id); + streamedFileIngestTasksQueue.add(task); } - refillIngestThreadQueue(); + refillFileIngestTasksQueue(); } } /** * Schedules file level ingest tasks for a given set of files for an ingest - * job pipeline by adding them directly to the front of the file tasks queue - * for the ingest manager's file ingest threads. + * job by adding them directly to the front of the file tasks queue consumed + * by the ingest manager's file ingest threads. This method is intended to + * be used to schedule files that are products of ingest module processing, + * e.g., extracted files and carved files. * - * @param ingestJobPipeline The ingestJobPipeline. - * @param files A set of files for the data source. + * Note that task scheduling for an ingest job and checking if all of the + * job's tasks are completed must always be atomic operations, enforced by + * synchronization using the scheduler's monitor. Otherwise, the completion + * checks could result in false positives. + * + * There is also a necessary convention that child tasks for products of + * other tasks in the job need to be scheduled before the parent tasks + * notify the scheduler that they are completed. This avoids false positives + * for completion checks by ensuring that the child tasks are queued before + * the parent tasks are marked as completed. + * + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. + * @param files The files. */ - synchronized void fastTrackFileIngestTasks(IngestPipeline ingestJobPipeline, Collection files) { - if (!ingestJobPipeline.isCancelled()) { + synchronized void fastTrackFileIngestTasks(IngestPipeline ingestPipeline, Collection files) { + if (!ingestPipeline.isCancelled()) { /* * Put the files directly into the queue for the file ingest * threads, if they pass the file filter for the job. The files are * added to the queue for the ingest threads BEFORE the other queued * tasks because the use case for this method is scheduling new - * carved or derived files from a higher priority task that is - * already in progress. + * carved or derived files from a high priority task that is already + * in progress. */ for (AbstractFile file : files) { - FileIngestTask fileTask = new FileIngestTask(ingestJobPipeline, file); + FileIngestTask fileTask = new FileIngestTask(ingestPipeline, file); if (shouldEnqueueFileTask(fileTask)) { try { - this.fileIngestThreadsQueue.putFirst(fileTask); + fileIngestTasksQueue.putFirst(fileTask); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while scheduling file level ingest tasks (jobId={%d)", ingestJobPipeline.getId()), ex); + DataSource dataSource = ingestPipeline.getDataSource(); + logger.log(Level.WARNING, String.format("Interrupted while enqueuing file tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS Thread.currentThread().interrupt(); return; } @@ -242,9 +330,26 @@ final class IngestTasksScheduler { } /** - * Schedules data artifact ingest tasks for a given ingest pipeline. + * Schedules data artifact ingest tasks for an ingest job for the data + * artifacts that have already been added to the case database. * - * @param ingestPipeline The ingest pipeline. + * Note that task scheduling for an ingest job and checking if all of the + * job's tasks are completed must always be atomic operations, enforced by + * synchronization using the scheduler's monitor. Otherwise, the completion + * checks could result in false positives. + * + * There is also a necessary convention that child tasks for products of + * other tasks in the job need to be scheduled before the parent tasks + * notify the scheduler that they are completed. This avoids false positives + * for completion checks by ensuring that the child tasks are queued before + * the parent tasks are marked as completed. + * + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. */ synchronized void scheduleDataArtifactIngestTasks(IngestPipeline ingestPipeline) { if (!ingestPipeline.isCancelled()) { @@ -253,26 +358,45 @@ final class IngestTasksScheduler { List artifacts = blackboard.getDataArtifacts(ingestPipeline.getDataSource()); scheduleDataArtifactIngestTasks(ingestPipeline, artifacts); } catch (TskCoreException ex) { - // RJCTODO + DataSource dataSource = ingestPipeline.getDataSource(); + logger.log(Level.SEVERE, String.format("Failed to retrieve data artifacts for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS } } } /** - * RJCTODO + * Schedules data artifact ingest tasks for an ingest job. This method is + * intended to be used to schedule artifacts that are products of ingest + * module processing. * - * @param ingestPipeline - * @param artifacts + * Note that task scheduling for an ingest job and checking if all of the + * job's tasks are completed must always be atomic operations, enforced by + * synchronization using the scheduler's monitor. Otherwise, the completion + * checks could result in false positives. + * + * There is also a necessary convention that child tasks for products of + * other tasks in the job need to be scheduled before the parent tasks + * notify the scheduler that they are completed. This avoids false positives + * for completion checks by ensuring that the child tasks are queued before + * the parent tasks are marked as completed. + * + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. + * @param artifacts The artifacts. */ synchronized void scheduleDataArtifactIngestTasks(IngestPipeline ingestPipeline, List artifacts) { // RJCTODO: WHy are cancellation checks in the scheduler instead of in the pipeline? if (!ingestPipeline.isCancelled()) { for (DataArtifact artifact : artifacts) { DataArtifactIngestTask task = new DataArtifactIngestTask(ingestPipeline, artifact); try { - this.dataArtifactIngestThreadQueue.putLast(task); + this.artifactIngestTasksQueue.putLast(task); } catch (InterruptedException ex) { - // RJCTODO: Better logging - IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data artifact ingest task queue (jobId={%d)", ingestPipeline.getId()), ex); + DataSource dataSource = ingestPipeline.getDataSource(); + logger.log(Level.WARNING, String.format("Interrupted while enqueuing data artifact tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS Thread.currentThread().interrupt(); break; } @@ -287,7 +411,7 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(DataSourceIngestTask task) { - dataSourceIngestThreadQueue.taskCompleted(task); + dataSourceIngestTasksQueue.taskCompleted(task); } /** @@ -297,8 +421,8 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(FileIngestTask task) { - fileIngestThreadsQueue.taskCompleted(task); - refillIngestThreadQueue(); + fileIngestTasksQueue.taskCompleted(task); + refillFileIngestTasksQueue(); } /** @@ -308,47 +432,56 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(DataArtifactIngestTask task) { - dataArtifactIngestThreadQueue.taskCompleted(task); - refillIngestThreadQueue(); + artifactIngestTasksQueue.taskCompleted(task); } /** * Queries the task scheduler to determine whether or not all of the ingest - * tasks for an ingest job pipeline have been completed. + * tasks for an ingest job have been completed. * - * @param ingestJobPipeline The ingestJobPipeline. + * @param ingestPipeline The ingest pipeline for the job. * * @return True or false. */ - synchronized boolean currentTasksAreCompleted(IngestPipeline ingestJobPipeline) { - long jobId = ingestJobPipeline.getId(); - - return !(this.dataSourceIngestThreadQueue.hasTasksForJob(jobId) - || hasTasksForJob(rootFileTaskQueue, jobId) - || hasTasksForJob(pendingFileTaskQueue, jobId) - || hasTasksForJob(streamedTasksQueue, jobId) - || fileIngestThreadsQueue.hasTasksForJob(jobId) - || dataArtifactIngestThreadQueue.hasTasksForJob(jobId)); + synchronized boolean currentTasksAreCompleted(IngestPipeline ingestPipeline) { + long pipelineId = ingestPipeline.getId(); + return !(dataSourceIngestTasksQueue.hasTasksForJob(pipelineId) + || hasTasksForJob(topLevelFileIngestTasksQueue, pipelineId) + || hasTasksForJob(batchedFileIngestTasksQueue, pipelineId) + || hasTasksForJob(streamedFileIngestTasksQueue, pipelineId) + || fileIngestTasksQueue.hasTasksForJob(pipelineId) + || artifactIngestTasksQueue.hasTasksForJob(pipelineId)); } /** - * Clears the "upstream" task scheduling queues for an ingest pipeline, but - * does nothing about tasks that have already been moved into the queue that - * is consumed by the file ingest threads. + * Cancels the pending file ingest tasks for an ingest job, where the + * pending tasks are the file ingest tasks that are in the upstream + * scheduling queues (batch and streaming) that feed into the queue consumed + * by the ingest manager's file ingest threads. * - * @param ingestJobPipeline The ingestJobPipeline. + * Note that the "normal" way to cancel an ingest job is to mark the job as + * cancelled, which causes the execute() methods of the ingest tasks for the + * job to return immediately when called, leading to flushing all of the + * tasks for the job out of the ingest task queues by the ingest threads and + * an orderly progression through IngestTaskTrackingQueue bookkeeping and + * the ingest job stages to early job completion. However, this method is a + * cancellation speed booster. For example, it eliminates the creation of + * what could be a large number of child tasks for both the top level files + * in the batch root file tasks queue and any directories in the batch root + * children file tasks queue. + * + * @param ingestJobPipeline The ingest pipeline for the job. */ - synchronized void cancelPendingTasksForIngestJob(IngestPipeline ingestJobPipeline) { + synchronized void cancelPendingFileTasksForIngestJob(IngestPipeline ingestJobPipeline) { long jobId = ingestJobPipeline.getId(); - IngestTasksScheduler.removeTasksForJob(rootFileTaskQueue, jobId); - IngestTasksScheduler.removeTasksForJob(pendingFileTaskQueue, jobId); - IngestTasksScheduler.removeTasksForJob(streamedTasksQueue, jobId); + removeTasksForJob(topLevelFileIngestTasksQueue, jobId); + removeTasksForJob(batchedFileIngestTasksQueue, jobId); + removeTasksForJob(streamedFileIngestTasksQueue, jobId); } /** - * Gets the top level files such as file system root directories, layout - * files and virtual directories for a data source. Used to create file - * tasks to put into the root directories queue. + * Gets the top level files for a data source, such as file system root + * directories, layout files, and virtual directories. * * @param dataSource The data source. * @@ -379,7 +512,8 @@ final class IngestTasksScheduler { } } } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS + // RJCTODO: Improve logging + logger.log(Level.SEVERE, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS } } } @@ -387,12 +521,15 @@ final class IngestTasksScheduler { } /** - * Schedules file ingest tasks for the ingest manager's file ingest threads. - * Files from streaming ingest will be prioritized. + * Refills the file ingest tasks queue consumed by the ingest manager's file + * ingest threads with tasks from the upstream file task scheduling queues + * (streamed and batch). Files from the streamed file ingest tasks queue are + * prioritized. Applies the file filter for the ingest job and attempts to + * move as many tasks as there are ingest threads. */ - synchronized private void refillIngestThreadQueue() { + synchronized private void refillFileIngestTasksQueue() { try { - takeFromStreamingTaskQueue(); + takeFromStreamingFileTasksQueue(); takeFromBatchTasksQueues(); } catch (InterruptedException ex) { IngestTasksScheduler.logger.log(Level.INFO, "Ingest tasks scheduler interrupted while blocked adding a task to the file level ingest task queue", ex); @@ -401,27 +538,21 @@ final class IngestTasksScheduler { } /** - * Move tasks from the streamedTasksQueue into the fileIngestThreadsQueue. - * Will attempt to move as many tasks as there are ingest threads. + * Moves tasks from the upstream streamed file ingest tasks queue into the + * file ingest tasks queue consumed by the ingest manager's file ingest + * threads. Applies the file filter for the ingest job and attempts to move + * as many tasks as there are ingest threads. */ - synchronized private void takeFromStreamingTaskQueue() throws InterruptedException { - /* - * Schedule files from the streamedTasksQueue - */ - while (fileIngestThreadsQueue.isEmpty()) { - /* - * We will attempt to schedule as many tasks as there are ingest - * queues. - */ + synchronized private void takeFromStreamingFileTasksQueue() throws InterruptedException { + while (fileIngestTasksQueue.isEmpty()) { int taskCount = 0; while (taskCount < IngestManager.getInstance().getNumberOfFileIngestThreads()) { - final FileIngestTask streamingTask = streamedTasksQueue.poll(); + final FileIngestTask streamingTask = streamedFileIngestTasksQueue.poll(); if (streamingTask == null) { return; // No streaming tasks are queued right now } - if (shouldEnqueueFileTask(streamingTask)) { - fileIngestThreadsQueue.putLast(streamingTask); + fileIngestTasksQueue.putLast(streamingTask); taskCount++; } } @@ -429,104 +560,91 @@ final class IngestTasksScheduler { } /** - * Schedules file ingest tasks for the ingest manager's file ingest threads - * by "shuffling" them through a sequence of three queues that allows for - * the interleaving of tasks from different data source ingest jobs based on - * priority, while limiting the number of queued tasks by only expanding - * directories one at a time. The sequence of queues is: + * Moves tasks from the upstream batched file ingest task queues into the + * file ingest tasks queue consumed by the ingest manager's file ingest + * threads. A sequence of two upstream queues is used to interleave tasks + * from different ingest jobs based on priority. Applies the file filter for + * the ingest job and attempts to move as many tasks as there are ingest + * threads. * - * 1. The root file tasks priority queue, which contains file tasks for the - * root objects of the data sources that are being analyzed. For example, - * the root tasks for a disk image data source are typically the tasks for - * the contents of the root directories of the file systems. This queue is a - * priority queue that attempts to ensure that user content is analyzed - * before general file system content. It feeds into the pending tasks - * queue. + * The upstream batched file task queues are: * - * 2. The pending file tasks queue, which contains root file tasks shuffled - * out of the root tasks queue, plus tasks for files with children - * discovered in the descent from the root tasks to the final leaf tasks in - * the content trees that are being analyzed for the data source ingest - * jobs. This queue is a FIFO queue that attempts to throttle the total - * number of file tasks by deferring queueing tasks for the children of - * files until the queue for the file ingest threads is emptied. It feeds - * into the file tasks queue for the ingest manager's file ingest threads. + * 1. The top level file tasks queue, which contains file tasks for the root + * objects of data sources. For example, the top level file tasks for a disk + * image data source are typically the tasks for the contents of the root + * directories of the file systems. This queue is a priority queue that + * attempts to ensure that user content is analyzed before general file + * system content. It feeds into the batched file ingest tasks queue. * - * 3. The file tasks queue for the ingest manager's file ingest threads. - * This queue is a blocking deque that is FIFO during a shuffle to maintain - * task prioritization, but LIFO when adding derived files to it directly - * during ingest. The reason for the LIFO additions is to give priority to - * files derived from prioritized files. + * 2. The batch file tasks queue, which contains top level file tasks moved + * in from the top level file tasks queue, plus tasks for child files in the + * descent from the root tasks to the final leaf tasks in the content trees + * that are being analyzed for any given data source. This queue is a FIFO + * queue that attempts to throttle the total number of file tasks by + * deferring queueing of tasks for the children of files until the queue for + * the file ingest threads is emptied. */ synchronized private void takeFromBatchTasksQueues() throws InterruptedException { - while (this.fileIngestThreadsQueue.isEmpty()) { + while (fileIngestTasksQueue.isEmpty()) { /* - * If the pending file task queue is empty, move the highest - * priority root file task, if there is one, into it. + * If the batched file task queue is empty, move the highest + * priority top level file task into it. */ - if (this.pendingFileTaskQueue.isEmpty()) { - final FileIngestTask rootTask = this.rootFileTaskQueue.pollFirst(); - if (rootTask != null) { - this.pendingFileTaskQueue.addLast(rootTask); + if (batchedFileIngestTasksQueue.isEmpty()) { + final FileIngestTask topLevelTask = topLevelFileIngestTasksQueue.pollFirst(); + if (topLevelTask != null) { + batchedFileIngestTasksQueue.addLast(topLevelTask); } } /* - * Try to move the next task from the pending task queue into the - * queue for the file ingest threads, if it passes the filter for - * the job. + * Try to move the next task from the batched file tasks queue into + * the queue for the file ingest threads. */ - final FileIngestTask pendingTask = this.pendingFileTaskQueue.pollFirst(); - if (pendingTask == null) { + final FileIngestTask nextTask = batchedFileIngestTasksQueue.pollFirst(); + if (nextTask == null) { return; } - if (shouldEnqueueFileTask(pendingTask)) { - /* - * The task is added to the queue for the ingest threads AFTER - * the higher priority tasks that preceded it. - */ - this.fileIngestThreadsQueue.putLast(pendingTask); + if (shouldEnqueueFileTask(nextTask)) { + fileIngestTasksQueue.putLast(nextTask); } /* * If the task that was just queued for the file ingest threads has - * children, try to queue tasks for the children. Each child task - * will go into either the directory queue if it has children of its - * own, or into the queue for the file ingest threads, if it passes - * the filter for the job. + * children, queue tasks for the children as well. */ AbstractFile file = null; try { - file = pendingTask.getFile(); + file = nextTask.getFile(); for (Content child : file.getChildren()) { if (child instanceof AbstractFile) { AbstractFile childFile = (AbstractFile) child; - FileIngestTask childTask = new FileIngestTask(pendingTask.getIngestJobPipeline(), childFile); + FileIngestTask childTask = new FileIngestTask(nextTask.getIngestPipeline(), childFile); if (childFile.hasChildren()) { - this.pendingFileTaskQueue.add(childTask); + batchedFileIngestTasksQueue.add(childTask); } else if (shouldEnqueueFileTask(childTask)) { - this.fileIngestThreadsQueue.putLast(childTask); + fileIngestTasksQueue.putLast(childTask); } } } } catch (TskCoreException ex) { if (file != null) { - logger.log(Level.SEVERE, String.format("Error getting the children of %s (objId=%d)", file.getName(), file.getId()), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Error getting the children of %s (object ID = %d)", file.getName(), file.getId()), ex); //NON-NLS } else { - // In practice, the task would have already returned false from the call to shouldEnqueueFileTask() - logger.log(Level.SEVERE, "Error loading file with object ID {0}", pendingTask.getFileId()); + logger.log(Level.SEVERE, "Error loading file with object ID = {0}", nextTask.getFileId()); //NON-NLS } } } } /** - * Examines the file associated with a file ingest task to determine whether - * or not the file should be processed and therefore whether or not the task - * should be enqueued. + * Evaluates the file for a file ingest task to determine whether or not the + * file should be processed and therefore whether or not the task should be + * enqueued. The evaluation includes applying the file filter for the task's + * parent ingest job. * - * @param task The task to be scrutinized. + * @param task The task. * * @return True or false. */ @@ -612,8 +730,7 @@ final class IngestTasksScheduler { } /** - * Check whether or not a file should be carved for a data source ingest - * ingest job. + * Checks whether or not a file should be carved for an ingest job. * * @param task The file ingest task for the file. * @@ -622,15 +739,15 @@ final class IngestTasksScheduler { private static boolean shouldBeCarved(final FileIngestTask task) { try { AbstractFile file = task.getFile(); - return task.getIngestJobPipeline().shouldProcessUnallocatedSpace() && file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS); + return task.getIngestPipeline().shouldProcessUnallocatedSpace() && file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS); } catch (TskCoreException ex) { return false; } } /** - * Checks whether or not a file is accepted (passes) the file filter for a - * data source ingest job. + * Checks whether or not a file is accepted (passes) the file filter for an + * ingest job. * * @param task The file ingest task for the file. * @@ -639,7 +756,7 @@ final class IngestTasksScheduler { private static boolean fileAcceptedByFilter(final FileIngestTask task) { try { AbstractFile file = task.getFile(); - return !(task.getIngestJobPipeline().getFileIngestFilter().fileIsMemberOf(file) == null); + return !(task.getIngestPipeline().getFileIngestFilter().fileIsMemberOf(file) == null); } catch (TskCoreException ex) { return false; } @@ -647,16 +764,16 @@ final class IngestTasksScheduler { /** * Checks whether or not a collection of ingest tasks includes a task for a - * given data source ingest job. + * given ingest job. * - * @param tasks The tasks. - * @param jobId The data source ingest job id. + * @param tasks The tasks. + * @param pipelineId The ID of the ingest pipeline for the job. * * @return True if there are no tasks for the job, false otherwise. */ - synchronized private static boolean hasTasksForJob(Collection tasks, long jobId) { + synchronized private static boolean hasTasksForJob(Collection tasks, long pipelineId) { for (IngestTask task : tasks) { - if (task.getIngestJobPipeline().getId() == jobId) { + if (task.getIngestPipeline().getId() == pipelineId) { return true; } } @@ -664,34 +781,35 @@ final class IngestTasksScheduler { } /** - * Removes all of the ingest tasks associated with a data source ingest job - * from a tasks collection. + * Removes all of the ingest tasks associated with an ingest job from a + * collection of tasks. * - * @param tasks The collection from which to remove the tasks. - * @param jobId The data source ingest job id. + * @param tasks The tasks. + * @param pipelineId The ID of the ingest pipeline for the job. */ - private static void removeTasksForJob(Collection tasks, long jobId) { + private static void removeTasksForJob(Collection tasks, long pipelineId) { Iterator iterator = tasks.iterator(); while (iterator.hasNext()) { IngestTask task = iterator.next(); - if (task.getIngestJobPipeline().getId() == jobId) { + if (task.getIngestPipeline().getId() == pipelineId) { // RJCTODO: Is this right? Why is there no synchronization, are these collections thread safe? iterator.remove(); } } } /** - * Counts the number of ingest tasks in a tasks collection for a given job. + * Counts the number of ingest tasks in a collection of tasks for a given + * ingest job. * - * @param queue The queue for which to count tasks. - * @param jobId The id of the job for which the tasks are to be counted. + * @param tasks The tasks. + * @param pipelineId The ID of the ingest pipeline for the job. * * @return The count. */ - private static int countTasksForJob(Collection queue, long jobId) { + private static int countTasksForJob(Collection tasks, long pipelineId) { int count = 0; - for (IngestTask task : queue) { - if (task.getIngestJobPipeline().getId() == jobId) { + for (IngestTask task : tasks) { + if (task.getIngestPipeline().getId() == pipelineId) { count++; } } @@ -699,6 +817,7 @@ final class IngestTasksScheduler { } /** + * //RJCTODO * Returns a snapshot of the states of the tasks in progress for an ingest * job. * @@ -707,14 +826,12 @@ final class IngestTasksScheduler { * @return */ synchronized IngestJobTasksSnapshot getTasksSnapshotForJob(long jobId) { - return new IngestJobTasksSnapshot(jobId, this.dataSourceIngestThreadQueue.countQueuedTasksForJob(jobId), - countTasksForJob(this.rootFileTaskQueue, jobId), - countTasksForJob(this.pendingFileTaskQueue, jobId), - this.fileIngestThreadsQueue.countQueuedTasksForJob(jobId), - this.dataSourceIngestThreadQueue.countRunningTasksForJob(jobId) + this.fileIngestThreadsQueue.countRunningTasksForJob(jobId), - countTasksForJob(this.streamedTasksQueue, jobId)); - // RJCTODO - + return new IngestJobTasksSnapshot(jobId, dataSourceIngestTasksQueue.countQueuedTasksForJob(jobId), + countTasksForJob(topLevelFileIngestTasksQueue, jobId), + countTasksForJob(batchedFileIngestTasksQueue, jobId), + fileIngestTasksQueue.countQueuedTasksForJob(jobId), + dataSourceIngestTasksQueue.countRunningTasksForJob(jobId) + fileIngestTasksQueue.countRunningTasksForJob(jobId), + countTasksForJob(streamedFileIngestTasksQueue, jobId)); } /** @@ -953,7 +1070,7 @@ final class IngestTasksScheduler { /** * Handles the completion of an ingest task by removing it from the - * running tasks list. + * tasks in progress list. * * @param task The completed task. */ @@ -965,43 +1082,41 @@ final class IngestTasksScheduler { /** * Checks whether there are any ingest tasks are queued and/or running - * for a given data source ingest job. + * for a given ingest job. * - * @param jobId The id of the data source ingest job. + * @param pipelineId The ID of the ingest pipeline for the job. * * @return */ - boolean hasTasksForJob(long jobId) { + boolean hasTasksForJob(long pipelineId) { synchronized (this) { - return IngestTasksScheduler.hasTasksForJob(this.queuedTasks, jobId) || IngestTasksScheduler.hasTasksForJob(this.tasksInProgress, jobId); + return IngestTasksScheduler.hasTasksForJob(queuedTasks, pipelineId) || IngestTasksScheduler.hasTasksForJob(tasksInProgress, pipelineId); } } /** - * Gets a count of the queued ingest tasks for a given data source - * ingest job. + * Gets a count of the queued ingest tasks for a given ingest job. * - * @param jobId + * @param pipelineId The ID of the ingest pipeline for the job. * * @return */ - int countQueuedTasksForJob(long jobId) { + int countQueuedTasksForJob(long pipelineId) { synchronized (this) { - return IngestTasksScheduler.countTasksForJob(this.queuedTasks, jobId); + return IngestTasksScheduler.countTasksForJob(queuedTasks, pipelineId); } } /** - * Gets a count of the running ingest tasks for a given data source - * ingest job. + * Gets a count of the running ingest tasks for a given ingest job. * - * @param jobId + * @param pipelineId The ID of the ingest pipeline for the job. * * @return */ - int countRunningTasksForJob(long jobId) { + int countRunningTasksForJob(long pipelineId) { synchronized (this) { - return IngestTasksScheduler.countTasksForJob(this.tasksInProgress, jobId); + return IngestTasksScheduler.countTasksForJob(tasksInProgress, pipelineId); } } From b3039c84209682370957e4cc716d865890e62f44 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 12 Mar 2021 16:04:42 -0500 Subject: [PATCH 06/63] 7332 interim check in --- .../CentralRepoDataArtifactIngestModule.java | 1 - .../autopsy/ingest/Bundle.properties | 1 + .../autopsy/ingest/Bundle.properties-MERGED | 1 + .../ingest/DataArtifactIngestModule.java | 8 + .../ingest/DataArtifactIngestPipeline.java | 47 +- .../ingest/DataArtifactIngestTask.java | 2 +- .../ingest/DataSourceIngestModule.java | 11 +- .../ingest/DataSourceIngestModuleAdapter.java | 15 +- .../DataSourceIngestModuleProgress.java | 4 +- .../ingest/DataSourceIngestPipeline.java | 10 +- .../autopsy/ingest/DataSourceIngestTask.java | 2 +- .../autopsy/ingest/FileIngestModule.java | 8 + .../ingest/FileIngestModuleAdapter.java | 16 +- .../autopsy/ingest/FileIngestPipeline.java | 47 +- .../autopsy/ingest/FileIngestTask.java | 8 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 26 +- .../autopsy/ingest/IngestJobContext.java | 33 +- ...stPipeline.java => IngestJobPipeline.java} | 539 +++++++++--------- .../autopsy/ingest/IngestManager.java | 6 +- .../autopsy/ingest/IngestModule.java | 9 +- .../autopsy/ingest/IngestModuleFactory.java | 62 +- .../ingest/IngestModuleFactoryAdapter.java | 2 +- .../sleuthkit/autopsy/ingest/IngestTask.java | 14 +- .../autopsy/ingest/IngestTaskPipeline.java | 109 ++-- .../autopsy/ingest/IngestTasksScheduler.java | 36 +- .../pictureanalyzer/impls/EXIFProcessor.java | 42 +- 26 files changed, 510 insertions(+), 549 deletions(-) rename Core/src/org/sleuthkit/autopsy/ingest/{IngestPipeline.java => IngestJobPipeline.java} (77%) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java index 278c6ba0e4..7ecaf9b46a 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java @@ -45,7 +45,6 @@ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestMo CentralRepoIngestModuleFactory.getModuleName(), "Data Artifacts Processed", //NON-NLS String.format("Count = %d", artifactCounter.get()))); //NON-NLS - } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index 4b5b431537..9f00ead0b0 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -20,6 +20,7 @@ IngestMessagePanel.totalUniqueMessagesNameVal.text=- IngestJob.progress.dataSourceIngest.initialDisplayName=Analyzing {0} IngestJob.progress.dataSourceIngest.displayName={0} for {1} IngestJob.progress.fileIngest.displayName=Analyzing files from {0} +IngestJob.progress.resultsIngest.displayName=Analyzing results from {0} IngestJob.progress.cancelling=Cancelling... IngestJob.cancellationDialog.title=Cancel Ingest IngestDialog.startButton.title=Start diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED index 157506a57f..90242fe29e 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED @@ -35,6 +35,7 @@ IngestMessagePanel.totalUniqueMessagesNameVal.text=- IngestJob.progress.dataSourceIngest.initialDisplayName=Analyzing {0} IngestJob.progress.dataSourceIngest.displayName={0} for {1} IngestJob.progress.fileIngest.displayName=Analyzing files from {0} +IngestJob.progress.resultsIngest.displayName=Analyzing results from {0} IngestJob.progress.cancelling=Cancelling... IngestJob.cancellationDialog.title=Cancel Ingest IngestDialog.startButton.title=Start diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java index f45654185c..a37816d6be 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java @@ -29,6 +29,14 @@ public interface DataArtifactIngestModule extends IngestModule { /** * Processes a data artifact. * + * IMPORTANT: In addition to returning ProcessResult.OK or + * ProcessResult.ERROR, modules should log all errors using methods provided + * by the org.sleuthkit.autopsy.coreutils.Logger class. Log messages should + * include the name and object ID of the data being processed. If an + * exception has been caught by the module, the exception should be sent to + * the Logger along with the log message so that a stack trace will appear + * in the application log. + * * @param artifact The artifact to process. * * @return A result code indicating success or failure of the processing. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java index 35382899c8..b8b7c95cd3 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java @@ -20,6 +20,11 @@ package org.sleuthkit.autopsy.ingest; import java.util.List; import java.util.Optional; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataArtifact; +import org.sleuthkit.datamodel.TskCoreException; /** * A pipeline of data artifact ingest modules for performing data artifact @@ -27,7 +32,7 @@ import java.util.Optional; */ final class DataArtifactIngestPipeline extends IngestTaskPipeline { - private final IngestPipeline ingestJobPipeline; + private static final Logger logger = Logger.getLogger(IngestJobPipeline.class.getName()); /** * Constructs a pipeline of data artifact ingest modules for performing data @@ -37,38 +42,39 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline moduleTemplates) { + DataArtifactIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { super(ingestJobPipeline, moduleTemplates); - this.ingestJobPipeline = ingestJobPipeline; } @Override - Optional> acceptModuleTemplate(IngestModuleTemplate ingestModuleTemplate) { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + Optional> acceptModuleTemplate(IngestModuleTemplate template) { + Optional> module = Optional.empty(); + if (template.isDataArtifactIngestModuleTemplate()) { + DataArtifactIngestModule ingestModule = template.createDataArtifactIngestModule(); + module = Optional.of(new DataArtifactIngestPipelineModule(ingestModule, template.getModuleName())); + } + return module; } @Override - void prepareTask(DataArtifactIngestTask task) throws IngestTaskPipelineException { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + void prepareForTask(DataArtifactIngestTask task) throws IngestTaskPipelineException { } @Override - void completeTask(DataArtifactIngestTask task) throws IngestTaskPipelineException { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + void cleanUpAfterTask(DataArtifactIngestTask task) throws IngestTaskPipelineException { } /** * A wrapper that adds ingest infrastructure operations to a data artifact * ingest module. */ - static final class DataArtifactIngestPipelineModule extends IngestTaskPipeline.PipelineModule { + static final class DataArtifactIngestPipelineModule extends IngestTaskPipeline.PipelineModule { private final DataArtifactIngestModule module; /** * Constructs a wrapper that adds ingest infrastructure operations to a - * file ingest module. - * + * data artifact ingest module. * * @param module The module. * @param displayName The display name of the module. @@ -78,18 +84,13 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,10 +30,19 @@ public interface DataSourceIngestModule extends IngestModule { * Processes a data source. Called once between calls to startUp() and * shutDown(). * + * IMPORTANT: In addition to returning ProcessResult.OK or + * ProcessResult.ERROR, modules should log all errors using methods provided + * by the org.sleuthkit.autopsy.coreutils.Logger class. Log messages should + * include the name and object ID of the data being processed. If an + * exception has been caught by the module, the exception should be sent to + * the Logger along with the log message so that a stack trace will appear + * in the application log. + * * @param dataSource The data source to process. * @param progressBar A progress bar to be used to report progress. * * @return A result code indicating success or failure of the processing. */ ProcessResult process(Content dataSource, DataSourceIngestModuleProgress progressBar); + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java index 83b5a1457f..f4f58e1503 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,20 +21,15 @@ package org.sleuthkit.autopsy.ingest; import org.sleuthkit.datamodel.Content; /** + * DO NOT USE: As of Java 8, interfaces can have default methods. IngestModule + * now provides default no-op versions of startUp() and shutDown(). This class + * is no longer needed and can be DEPRECATED when convenient. + * * An adapter that provides a no-op implementation of the startUp() method for * data source ingest modules. - * - * NOTE: As of Java 8, interfaces can have default methods. - * DataSourceIngestModule now provides default no-op versions of startUp() and - * shutDown(). This class is no longer needed and can be deprecated when - * convenient. */ public abstract class DataSourceIngestModuleAdapter implements DataSourceIngestModule { - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - } - @Override public abstract ProcessResult process(Content dataSource, DataSourceIngestModuleProgress progressBar); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java index e13b2b7bd7..a2bd23b692 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java @@ -23,9 +23,9 @@ package org.sleuthkit.autopsy.ingest; */ public class DataSourceIngestModuleProgress { - private final IngestPipeline ingestJobPipeline; + private final IngestJobPipeline ingestJobPipeline; - DataSourceIngestModuleProgress(IngestPipeline pipeline) { + DataSourceIngestModuleProgress(IngestJobPipeline pipeline) { this.ingestJobPipeline = pipeline; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 3c5fc2f98f..b0703f21d6 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -26,7 +26,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; /** - * A pipeline of data source level ingest modules for performing data source + * A pipeline of data source level ingest modules for executing data source * level ingest tasks for an ingest job. */ final class DataSourceIngestPipeline extends IngestTaskPipeline { @@ -42,7 +42,7 @@ final class DataSourceIngestPipeline extends IngestTaskPipeline moduleTemplates) { + DataSourceIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { super(ingestJobPipeline, moduleTemplates); } @@ -57,11 +57,11 @@ final class DataSourceIngestPipeline extends IngestTaskPipeline sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,23 +21,17 @@ package org.sleuthkit.autopsy.ingest; import org.sleuthkit.datamodel.AbstractFile; /** + * DO NOT USE: As of Java 8, interfaces can have default methods. IngestModule + * now provides default no-op versions of startUp() and shutDown(). This class + * is no longer needed and can be DEPRECATED when convenient. + * * * An adapter that provides no-op implementations of the startUp() and * shutDown() methods for file ingest modules. * - * NOTE: As of Java 8, interfaces can have default methods. FileIngestModule now - * provides default no-op versions of startUp() and shutDown(). This class is no - * longer needed and can be deprecated when convenient. */ public abstract class FileIngestModuleAdapter implements FileIngestModule { - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - } - @Override public abstract ProcessResult process(AbstractFile file); - @Override - public void shutDown() { - } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index d957077403..4fabe12331 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -24,34 +24,27 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * A pipeline of file ingest modules for performing file ingest tasks for an + * A pipeline of file ingest modules for executing file ingest tasks for an * ingest job. */ final class FileIngestPipeline extends IngestTaskPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestPipeline ingestJobPipeline; + private final IngestJobPipeline ingestJobPipeline; /** - * Constructs a pipeline of file ingest modules for performing file ingest + * Constructs a pipeline of file ingest modules for executing file ingest * tasks for an ingest job. * * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. * @param moduleTemplates The ingest module templates that define this * pipeline. */ - FileIngestPipeline(IngestPipeline ingestJobPipeline, List moduleTemplates) { + FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { super(ingestJobPipeline, moduleTemplates); this.ingestJobPipeline = ingestJobPipeline; } - /** - * RJCTODO - * - * @param template - * - * @return - */ @Override Optional> acceptModuleTemplate(IngestModuleTemplate template) { Optional> module = Optional.empty(); @@ -62,28 +55,12 @@ final class FileIngestPipeline extends IngestTaskPipeline { return module; } - /** - * RJCTODO - * - * @param task - * - * @throws - * org.sleuthkit.autopsy.ingest.IngestTaskPipeline.IngestTaskPipelineException - */ @Override - void prepareTask(FileIngestTask task) throws IngestTaskPipelineException { + void prepareForTask(FileIngestTask task) throws IngestTaskPipelineException { } - /** - * RJCTODO - * - * @param task - * - * @throws - * org.sleuthkit.autopsy.ingest.IngestTaskPipeline.IngestTaskPipelineException - */ @Override - void completeTask(FileIngestTask task) throws IngestTaskPipelineException { + void cleanUpAfterTask(FileIngestTask task) throws IngestTaskPipelineException { AbstractFile file = null; try { file = task.getFile(); @@ -130,17 +107,8 @@ final class FileIngestPipeline extends IngestTaskPipeline { this.module = module; } - /** - * RJCTODO - * - * @param ingestJobPipeline - * @param task - * - * @throws - * org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException - */ @Override - void performTask(IngestPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException { + void executeTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException { AbstractFile file = null; try { file = task.getFile(); @@ -148,7 +116,6 @@ final class FileIngestPipeline extends IngestTaskPipeline { throw new IngestModuleException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS } ingestManager.setIngestTaskProgress(task, getDisplayName()); - ingestJobPipeline.setCurrentFileIngestModule(getDisplayName(), file.getName()); ProcessResult result = module.process(file); if (result == ProcessResult.ERROR) { throw new IngestModuleException(String.format("%s experienced an error analyzing %s (file objId = %d)", getDisplayName(), file.getName(), file.getId())); //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 76769e893e..48ae2d9820 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -40,7 +40,7 @@ final class FileIngestTask extends IngestTask { * task. * @param file The file to be processed. */ - FileIngestTask(IngestPipeline ingestJobPipeline, AbstractFile file) { + FileIngestTask(IngestJobPipeline ingestJobPipeline, AbstractFile file) { super(ingestJobPipeline); this.file = file; fileId = file.getId(); @@ -56,7 +56,7 @@ final class FileIngestTask extends IngestTask { * task. * @param fileId The object ID of the file to be processed. */ - FileIngestTask(IngestPipeline ingestJobPipeline, long fileId) { + FileIngestTask(IngestJobPipeline ingestJobPipeline, long fileId) { super(ingestJobPipeline); file = null; this.fileId = fileId; @@ -95,8 +95,8 @@ final class FileIngestTask extends IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - IngestPipeline thisPipeline = getIngestPipeline(); - IngestPipeline otherPipeline = other.getIngestPipeline(); + IngestJobPipeline thisPipeline = getIngestPipeline(); + IngestJobPipeline otherPipeline = other.getIngestPipeline(); if (thisPipeline != otherPipeline && (thisPipeline == null || !thisPipeline.equals(otherPipeline))) { return false; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 20b8b65592..46d44bc7ab 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2020 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,7 +33,6 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.DataSource; /** * Analyzes one or more data sources using a set of ingest modules specified via @@ -78,7 +77,7 @@ public final class IngestJob { private final List dataSources = new ArrayList<>(); private final List files = new ArrayList<>(); private final Mode ingestMode; - private final Map ingestJobPipelines; + private final Map ingestJobPipelines; private final AtomicInteger incompleteJobsCount; private final IngestJobSettings settings; private volatile CancellationReason cancellationReason; @@ -136,7 +135,6 @@ public final class IngestJob { * @return The job identifier. */ public long getId() { - // RJCTODO: Replace this with the pipeline ID, as has been done elsewhere. return this.id; } @@ -162,7 +160,7 @@ public final class IngestJob { return; } // Streaming ingest jobs will only have one data source - IngestPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); + IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); streamingIngestPipeline.addStreamingIngestFiles(fileObjIds); } @@ -175,7 +173,7 @@ public final class IngestJob { return; } // Streaming ingest jobs will only have one data source - IngestPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); + IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); streamingIngestPipeline.addStreamingIngestDataSource(); } @@ -193,11 +191,11 @@ public final class IngestJob { */ if (files.isEmpty()) { for (Content dataSource : dataSources) { - IngestPipeline ingestJobPipeline = new IngestPipeline(this, dataSource, settings); + IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSource, settings); ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); } } else { - IngestPipeline ingestJobPipeline = new IngestPipeline(this, dataSources.get(0), files, settings); + IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSources.get(0), files, settings); ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); } incompleteJobsCount.set(ingestJobPipelines.size()); @@ -206,7 +204,7 @@ public final class IngestJob { * Try to start up each ingest job pipeline. Stop at the first failure. */ List errors = new ArrayList<>(); - for (IngestPipeline ingestJobPipeline : ingestJobPipelines.values()) { + for (IngestJobPipeline ingestJobPipeline : ingestJobPipelines.values()) { errors.addAll(ingestJobPipeline.startUp()); if (errors.isEmpty() == false) { break; @@ -217,7 +215,7 @@ public final class IngestJob { * Handle start up success or failure. */ if (errors.isEmpty()) { - for (IngestPipeline ingestJobPipeline : ingestJobPipelines.values()) { + for (IngestJobPipeline ingestJobPipeline : ingestJobPipelines.values()) { IngestManager.getInstance().fireDataSourceAnalysisStarted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource()); } } else { @@ -323,7 +321,7 @@ public final class IngestJob { * * @param ingestJobPipeline A completed ingestJobPipeline. */ - void notifyIngestPipelineShutDown(IngestPipeline ingestJobPipeline) { + void notifyIngestPipelineShutDown(IngestJobPipeline ingestJobPipeline) { IngestManager ingestManager = IngestManager.getInstance(); if (!ingestJobPipeline.isCancelled()) { ingestManager.fireDataSourceAnalysisCompleted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource()); @@ -409,7 +407,7 @@ public final class IngestJob { fileIngestRunning = false; fileIngestStartTime = null; dataSourceProcessingSnapshots = new ArrayList<>(); - for (IngestPipeline pipeline : ingestJobPipelines.values()) { + for (IngestJobPipeline pipeline : ingestJobPipelines.values()) { Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot); dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot)); if (null == dataSourceModule) { @@ -496,7 +494,7 @@ public final class IngestJob { */ public static class DataSourceIngestModuleHandle { - private final IngestPipeline ingestJobPipeline; + private final IngestJobPipeline ingestJobPipeline; private final DataSourceIngestPipeline.DataSourcePipelineModule module; private final boolean cancelled; @@ -509,7 +507,7 @@ public final class IngestJob { * source level ingest module. * @param module The data source level ingest module. */ - private DataSourceIngestModuleHandle(IngestPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) { + private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) { this.ingestJobPipeline = ingestJobPipeline; this.module = module; this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 65b660dc8e..62b531e541 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.ingest; +import java.util.ArrayList; import java.util.List; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; @@ -32,7 +33,7 @@ import org.sleuthkit.datamodel.DataArtifact; */ public final class IngestJobContext { - private final IngestPipeline ingestPipeline; + private final IngestJobPipeline ingestPipeline; /** * Constructs an ingest job context object that provides an ingest module @@ -40,7 +41,7 @@ public final class IngestJobContext { * * @param ingestJobPipeline */ - IngestJobContext(IngestPipeline ingestJobPipeline) { + IngestJobContext(IngestJobPipeline ingestJobPipeline) { this.ingestPipeline = ingestJobPipeline; } @@ -50,7 +51,7 @@ public final class IngestJobContext { * @return The context string. */ public String getExecutionContext() { - return this.ingestPipeline.getExecutionContext(); + return ingestPipeline.getExecutionContext(); } /** @@ -89,13 +90,31 @@ public final class IngestJobContext { * check this periodically and break off processing if the method returns * true. * - * RJCTODO: Figure out how to check/handle cancellation for data artoifact - * ingest modules. Perhaps it is time to add a cancel method with an - * implementation in teh "adapter" classes. - * * @return True or false. */ public boolean fileIngestIsCancelled() { + /* + * It is not currently possible to cancel individual file ingest + * modules. File ingest cancellation is equiovalent to ingest job + * cancellation. + */ + return ingestPipeline.isCancelled(); + } + + /** + * Checks whether or not cancellation of the currently running data artifact + * ingest module for the ingest job has been requested. File ingest modules + * should check this periodically and break off processing if the method + * returns true. + * + * @return True or false. + */ + public boolean dataArtifactIngestIsCancelled() { + /* + * It is not currently possible to cancel individual file ingest + * modules. Data artifact ingest cancellation is equivalent to ingest + * job cancellation. + */ return ingestPipeline.isCancelled(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java similarity index 77% rename from Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java rename to Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index e76555d9e4..f5354503ed 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -37,7 +37,6 @@ import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; @@ -58,32 +57,43 @@ import org.sleuthkit.datamodel.DataSource; /** * An ingest pipeline that works with the ingest tasks scheduler to coordinate * the creation, scheduling, and execution of ingest tasks for one of the data - * sources in an ingest job. An ingest pipeline has one to many child ingest - * task pipelines. An ingest task pipeline is a sequences of ingest modules of a + * sources in an ingest job. An ingest job pipeline has one-to-many child ingest + * task pipelines. An ingest task pipeline is a sequence of ingest modules of a * given type that have been enabled and configured as part of the settings for * the ingest job. */ -final class IngestPipeline { +final class IngestJobPipeline { - private static String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy"; /* * The class names of the proxy classes Jython generates for Python classes * look something like: * "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" */ private static final Pattern JYTHON_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); - private static final Logger logger = Logger.getLogger(IngestPipeline.class.getName()); - private static final AtomicLong nextPipelineId = new AtomicLong(0L); + private static final String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy"; - /** + private static final Logger logger = Logger.getLogger(IngestJobPipeline.class.getName()); + private final IngestJob job; + private final IngestJobSettings settings; + private DataSource dataSource; + private final List files; + private final long createTime; + + /* * An ingest pipeline interacts with the ingest task scheduler to schedule * initial ingest tasks, determine whether or not there are ingest tasks * still to be executed, and to schedule additional tasks submitted by * ingest modules. For example, a file carving module can add carved files * to an ingest job and many modules will add data artifacts to an ingest * job. + * + * The pipeline ID is used to associate the pipeline with its tasks. The + * ingest job ID cannot be used for this because an ingest job may have more + * than one pipeline (one pipeline per data source). */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); + private static final AtomicLong nextPipelineId = new AtomicLong(0L); + private final long pipelineId; /* * An ingest pipeline runs its child ingest task pipelines in stages. @@ -95,30 +105,20 @@ final class IngestPipeline { */ INITIALIZATION, /* - * The ingest pipeline is only running its ingest task pipelines that - * process results such as data artifacts, OS accounts, etc., that are - * generated by some other part of the ingest process. Typically, but - * not always, this will be a transitional stage that happens first - * because the results pipelines need to be started up before the file - * and data source level pipelines, so that results generated by those - * pipelines can be processed by the results pipelines. - */ - FIRST_STAGE_RESULTS_TASKS_ONLY, - /* - * The ingest pipeline is only running its file and results ingest task + * The ingest pipeline is only running its file and result ingest task * pipelines because the data source for the job has not been supplied * to it yet. This stage is only used for streaming ingest. */ FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY, /* * The ingest pipeline is running its high priority data source level - * ingest task pipeline, its file ingest pipeline, and its results - * pipelines. + * ingest task pipeline, its file ingest task pipelines, and its result + * ingest task pipelines. */ FIRST_STAGE_ALL_TASKS, /** * The ingest pipeline is running its lower priority data source level - * ingest task pipeline and its results task pipelines. + * ingest task pipeline and its result task pipelines. */ SECOND_STAGE, /** @@ -126,15 +126,7 @@ final class IngestPipeline { */ FINALIZATION }; - - private final IngestJob job; - private final long pipelineId; - private final IngestJobSettings settings; - private DataSource dataSource; - private final IngestJob.Mode ingestMode; - private final List files = new ArrayList<>(); - private final long createTime; - private volatile Stages stage = IngestPipeline.Stages.INITIALIZATION; + private volatile Stages stage = IngestJobPipeline.Stages.INITIALIZATION; private final Object stageTransitionLock = new Object(); /* @@ -162,8 +154,9 @@ final class IngestPipeline { /** * An ingest pipeline has a collection of identical file ingest task - * pipelines, one for each file ingest thread in the ingest manager. The - * ingest threads take and return task pipelines using a blocking queue. + * pipelines, one for each file ingest thread in the ingest manager. + * + * The ingest threads take and return task pipelines using a blocking queue. * Additionally, a fixed list of all of the file pipelines is used to allow * cycling through each of the individual task pipelines to check their * status. @@ -205,33 +198,30 @@ final class IngestPipeline { private long estimatedFilesToProcess; private long processedFiles; private ProgressHandle fileIngestProgress; - private String currentFileIngestModule = ""; - private String currentFileIngestTask = ""; - private final List ingestModules = new ArrayList<>(); private volatile IngestJobInfo ingestJobInfo; /** * Constructs an ingest pipeline that works with the ingest tasks scheduler * to coordinate the creation, scheduling, and execution of ingest tasks for - * one of the data sources in an ingest job. An ingest pipeline has one to - * many child ingest task pipelines. The child ingest task pipelines are - * sequences of ingest modules of a given type that have been enabled and + * one of the data sources in an ingest job. An ingest job pipeline has + * one-to-many child ingest task pipelines. An ingest task pipeline is a + * sequence of ingest modules of a given type that have been enabled and * configured as part of the settings for the ingest job. * * @param job The ingest job. * @param dataSource The data source. * @param settings The ingest settings for the ingest job. */ - IngestPipeline(IngestJob job, Content dataSource, IngestJobSettings settings) { + IngestJobPipeline(IngestJob job, Content dataSource, IngestJobSettings settings) { this(job, dataSource, Collections.emptyList(), settings); } /** * Constructs an ingest pipeline that works with the ingest tasks scheduler * to coordinate the creation, scheduling, and execution of ingest tasks for - * one of the data sources in an ingest job. An ingest pipeline has one to - * many child ingest task pipelines. The child ingest task pipelines are - * sequences of ingest modules of a given type that have been enabled and + * one of the data sources in an ingest job. An ingest job pipeline has + * one-to-many child ingest task pipelines. An ingest task pipeline is a + * sequence of ingest modules of a given type that have been enabled and * configured as part of the settings for the ingest job. * * @param job The ingest job. @@ -241,21 +231,19 @@ final class IngestPipeline { * processed. * @param settings The ingest settings for the ingest job. */ - IngestPipeline(IngestJob job, Content dataSource, List files, IngestJobSettings settings) { + IngestJobPipeline(IngestJob job, Content dataSource, List files, IngestJobSettings settings) { if (!(dataSource instanceof DataSource)) { throw new IllegalArgumentException("Passed dataSource that does not implement the DataSource interface"); //NON-NLS } this.job = job; - this.dataSource = (DataSource) dataSource; this.settings = settings; + this.dataSource = (DataSource) dataSource; + this.files = new ArrayList<>(); this.files.addAll(files); - pipelineId = IngestPipeline.nextPipelineId.getAndIncrement(); - ingestMode = job.getIngestMode(); + pipelineId = IngestJobPipeline.nextPipelineId.getAndIncrement(); doUI = RuntimeProperties.runningWithGUI(); createTime = new Date().getTime(); stage = Stages.INITIALIZATION; - currentFileIngestModule = ""; //NON-NLS - currentFileIngestTask = ""; //NON-NLS createIngestTaskPipelines(); } @@ -269,7 +257,7 @@ final class IngestPipeline { * @param jythonModules The input ingest module templates for modules * implemented using Jython. */ - private static void addOrdered(final List orderedModules, final Map javaModules, final Map jythonModules) { + private static void completePipeline(final List orderedModules, final Map javaModules, final Map jythonModules) { final List autopsyModules = new ArrayList<>(); final List thirdPartyModules = new ArrayList<>(); Stream.concat(javaModules.entrySet().stream(), jythonModules.entrySet().stream()).forEach((templateEntry) -> { @@ -313,7 +301,7 @@ final class IngestPipeline { * @param jythonMapping Mapping for Jython ingest module templates. * @param template The ingest module template. */ - private static void addModule(Map mapping, Map jythonMapping, IngestModuleTemplate template) { + private static void addIngestModuleTermplateToMaps(Map mapping, Map jythonMapping, IngestModuleTemplate template) { String className = template.getModuleFactory().getClass().getCanonicalName(); String jythonName = getModuleNameFromJythonClassName(className); if (jythonName != null) { @@ -328,64 +316,75 @@ final class IngestPipeline { */ private void createIngestTaskPipelines() { /* - * Get the complete set of ingest module templates from the ingest job - * settings. An ingest module template combines an ingest module factory - * with job level ingest module settings to support the creation of any - * number of fully configured instances of a given ingest module. + * Get the enabled ingest module templates from the ingest job settings. + * An ingest module template combines an ingest module factory with job + * level ingest module settings to support the creation of any number of + * fully configured instances of a given ingest module. + * + * Note that an ingest module factory may be able to create multiple + * tpyes of ingest modules. */ - List enabledIngestModuleTemplates = this.settings.getEnabledIngestModuleTemplates(); + List enabledTemplates = settings.getEnabledIngestModuleTemplates(); /** - * Make one mapping of ingest module factory class names to ingest - * module templates for each type of ingest task pipeline to be created. - * These mappings are used to go from an ingest module factory class - * name in the pipeline configuration file to the corresponding ingest - * module template. + * Separate the ingest module templates into buckets based on the module + * types the ingest module factory can create. A template may go into + * more than one bucket. The buckets are actually maps of ingest module + * factory class names to ingest module templates. The maps are used to + * go from an ingest module factory class name read from the pipeline + * configuration file to the corresponding ingest module template. + * + * There are also two maps for each bucket. One map is for Java modules + * and the other one is for Jython modules. The templates are separated + * this way so that Java modules that are not in the pipeline config + * file can be placed before Jython modules. */ - Map unorderedDataSourceModuleTemplates = new LinkedHashMap<>(); - Map unorderedJythonDataSourceModuleTemplates = new LinkedHashMap<>(); - Map unorderedFileModuleTemplates = new LinkedHashMap<>(); - Map unorderedJythonFileModuleTemplates = new LinkedHashMap<>(); - Map unorderedArtifactModuleTemplates = new LinkedHashMap<>(); - Map unorderedJythonArtifactModuleTemplates = new LinkedHashMap<>(); - for (IngestModuleTemplate template : enabledIngestModuleTemplates) { + Map javaDataSourceModuleTemplates = new LinkedHashMap<>(); + Map jythonDataSourceModuleTemplates = new LinkedHashMap<>(); + Map javaFileModuleTemplates = new LinkedHashMap<>(); + Map jythonFileModuleTemplates = new LinkedHashMap<>(); + Map javaArtifactModuleTemplates = new LinkedHashMap<>(); + Map jythonArtifactModuleTemplates = new LinkedHashMap<>(); + for (IngestModuleTemplate template : enabledTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - addModule(unorderedDataSourceModuleTemplates, unorderedJythonDataSourceModuleTemplates, template); - continue; + addIngestModuleTermplateToMaps(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); } if (template.isFileIngestModuleTemplate()) { - addModule(unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates, template); - continue; + addIngestModuleTermplateToMaps(javaFileModuleTemplates, jythonFileModuleTemplates, template); } if (template.isDataArtifactIngestModuleTemplate()) { - addModule(unorderedArtifactModuleTemplates, unorderedJythonArtifactModuleTemplates, template); + addIngestModuleTermplateToMaps(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); } } /** - * Use the mappings and the entries read from the ingest pipelines - * configuration file to create ordered lists of ingest module template - * for each ingest task pipeline. + * Take the module templates that have pipeline configuration file + * entries out of the buckets and put them in lists representing ingest + * task pipelines, in the order prescribed by the file. + * + * Note that the pipeline configuration file currently only supports + * specifying data source level and file ingest module pipeline layouts. */ - IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = IngestPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, unorderedJythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = IngestPipeline.getConfiguredIngestModuleTemplates(unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = IngestPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + IngestPipelinesConfiguration pipelineConfig = IngestPipelinesConfiguration.getInstance(); + List firstStageDataSourceModuleTemplates = createPipelineFromConfigFile(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageOneDataSourceIngestPipelineConfig()); + List secondStageDataSourceModuleTemplates = createPipelineFromConfigFile(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageTwoDataSourceIngestPipelineConfig()); + List fileIngestModuleTemplates = createPipelineFromConfigFile(javaFileModuleTemplates, jythonFileModuleTemplates, pipelineConfig.getFileIngestPipelineConfig()); + List artifactModuleTemplates = new ArrayList<>(); /** - * Add any module templates that were not specified in the pipeline - * configuration file to the appropriate list. Note that no data - * artifact ingest modules are currently specified in the file and that - * core Autopsy modules are added first and third party modules are - * added last. + * Add any module templates remaining in the buckets to the appropriate + * ingest task pipeline. Note that any data source level ingest modules + * that were not listed in the configuration file are added to the first + * stage data source pipeline, Java modules are added before Jython + * modules, and Core Autopsy modules are added before third party + * modules. */ - addOrdered(firstStageDataSourceModuleTemplates, unorderedDataSourceModuleTemplates, unorderedJythonDataSourceModuleTemplates); - addOrdered(fileIngestModuleTemplates, unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates); - addOrdered(fileIngestModuleTemplates, unorderedFileModuleTemplates, unorderedJythonFileModuleTemplates); - List artifactModuleTemplates = IngestPipeline.getConfiguredIngestModuleTemplates(unorderedDataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + completePipeline(firstStageDataSourceModuleTemplates, javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates); + completePipeline(fileIngestModuleTemplates, javaFileModuleTemplates, jythonFileModuleTemplates); + completePipeline(artifactModuleTemplates, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); /** - * Construct the ingest task pipelines from the ordered lists. + * Construct the actual ingest task pipelines from the ordered lists. */ firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); @@ -393,48 +392,18 @@ final class IngestPipeline { int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); for (int i = 0; i < numberOfFileIngestThreads; ++i) { FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates); - this.fileIngestPipelinesQueue.put(pipeline); - this.fileIngestPipelines.add(pipeline); + fileIngestPipelinesQueue.put(pipeline); + fileIngestPipelines.add(pipeline); } } catch (InterruptedException ex) { - /** - * The current thread was interrupted while blocked on a full queue. - * Blocking should actually never happen here, but reset the - * interrupted flag rather than just swallowing the exception. + /* + * RJCTODO: This is incorrect. If this thread is interrupted, the + * pipeline is incomplete and should not be used. Either fix this or + * write a story about it. */ Thread.currentThread().interrupt(); } artifactIngestPipeline = new DataArtifactIngestPipeline(this, artifactModuleTemplates); - - /* - * Add the ingest module templates to the "master list" and record the - * final composition of the ingest task pipelines in the case database. - */ - try { - SleuthkitCase skCase = Case.getCurrentCaseThrows().getSleuthkitCase(); - addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); - addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase); - addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); - } catch (TskCoreException | NoCurrentCaseException ex) { - logErrorMessage(Level.WARNING, "Failed to add ingest modules listing to case database", ex); //NON-NLS - } - } - - /** - * Adds a list of ingest module templates for an ingest task pipeline to the - * modules "master list" and records the final composition of the pipeline - * in the case database. - * - * @param templates A list of ingest module templates. - * @param type The type of the ingest module templates. - * @param skCase The case database. - * - * @throws TskCoreException - */ - private void addIngestModules(List templates, IngestModuleType type, SleuthkitCase skCase) throws TskCoreException { - for (IngestModuleTemplate module : templates) { - ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber())); - } } /** @@ -458,12 +427,12 @@ final class IngestPipeline { * @return An ordered list of ingest module templates, i.e., an * uninstantiated pipeline. */ - private static List getConfiguredIngestModuleTemplates(Map javaIngestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { + private static List createPipelineFromConfigFile(Map javaIngestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { List templates = new ArrayList<>(); for (String moduleClassName : pipelineConfig) { - if (javaIngestModuleTemplates != null && javaIngestModuleTemplates.containsKey(moduleClassName)) { + if (javaIngestModuleTemplates.containsKey(moduleClassName)) { templates.add(javaIngestModuleTemplates.remove(moduleClassName)); - } else if (jythonIngestModuleTemplates != null && jythonIngestModuleTemplates.containsKey(moduleClassName)) { + } else if (jythonIngestModuleTemplates.containsKey(moduleClassName)) { templates.add(jythonIngestModuleTemplates.remove(moduleClassName)); } } @@ -589,7 +558,7 @@ final class IngestPipeline { if (errors.isEmpty()) { recordIngestJobStartUpInfo(); if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules()) { - if (ingestMode == IngestJob.Mode.BATCH) { + if (job.getIngestMode() == IngestJob.Mode.BATCH) { startFirstStageInBatchMode(); } else { startFirstStageInStreamingMode(); @@ -603,22 +572,54 @@ final class IngestPipeline { /** * Writes start up data about the ingest job into the case database. The - * case database returns a an object that is retained to allow the additon - * of a completion time when the ingest job is finished. - * - * This is here, rather than in the IngestJob class, because the logged data - * includes ordered lists of the ingest modules in the child ingest task - * pipelines. + * case database returns an object that is retained to allow the additon of + * a completion time when the ingest job is finished. */ void recordIngestJobStartUpInfo() { try { - SleuthkitCase caseDb = Case.getCurrentCaseThrows().getSleuthkitCase(); - ingestJobInfo = caseDb.addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); - } catch (TskCoreException | NoCurrentCaseException ex) { - logErrorMessage(Level.WARNING, "Failed to add ingest job info to case database", ex); //NON-NLS + SleuthkitCase caseDb = Case.getCurrentCase().getSleuthkitCase(); + List ingestModuleInfoList = new ArrayList<>(); + for (IngestModuleTemplate module : settings.getEnabledIngestModuleTemplates()) { + IngestModuleType moduleType = getIngestModuleTemplateType(module); + IngestModuleInfo moduleInfo = caseDb.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), moduleType, module.getModuleFactory().getModuleVersionNumber()); + ingestModuleInfoList.add(moduleInfo); + } + ingestJobInfo = caseDb.addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModuleInfoList, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); + } catch (TskCoreException ex) { + logErrorMessage(Level.SEVERE, "Failed to add ingest job info to case database", ex); //NON-NLS } } + /** + * Determines the type of ingest modules a given ingest module template + * supports. + * + * @param moduleTemplate The ingest module template. + * + * @return The ingest module type. may be IngestModuleType.MULTIPLE. + */ + private IngestModuleType getIngestModuleTemplateType(IngestModuleTemplate moduleTemplate) { + IngestModuleType type = null; + if (moduleTemplate.isDataSourceIngestModuleTemplate()) { + type = IngestModuleType.DATA_SOURCE_LEVEL; + } + if (moduleTemplate.isFileIngestModuleTemplate()) { + if (type == null) { + type = IngestModuleType.FILE_LEVEL; + } else { + type = IngestModuleType.MULTIPLE; + } + } + if (moduleTemplate.isDataArtifactIngestModuleTemplate()) { + if (type == null) { + type = IngestModuleType.DATA_ARTIFACT; + } else { + type = IngestModuleType.MULTIPLE; + } + } + return type; + } + /** * Starts up each of the child ingest task pipelines in this ingest * pipeline. @@ -664,14 +665,6 @@ final class IngestPipeline { if (!startUpErrors.isEmpty()) { List shutDownErrors = pipeline.shutDown(); if (!shutDownErrors.isEmpty()) { - /* - * The start up errors will ultimately be reported to the user - * for possible remedy, but the shut down errors are logged - * here. - * - * RJCTODO: Is this right? What is our contract with the module - * developers when start up fails? - */ logIngestModuleErrors(shutDownErrors); } } @@ -825,7 +818,7 @@ final class IngestPipeline { synchronized (fileIngestProgressLock) { estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); if (doUI && fileIngestProgress != null) { - fileIngestProgress.switchToDeterminate((int) estimatedFilesToProcess); // RJCTODO: Check assumption that bar start indeterminate + fileIngestProgress.switchToDeterminate((int) estimatedFilesToProcess); } } @@ -849,8 +842,8 @@ final class IngestPipeline { logInfoMessage("Scheduling first stage data source level ingest task in streaming mode"); //NON-NLS synchronized (this.stageTransitionLock) { - stage = IngestPipeline.Stages.FIRST_STAGE_ALL_TASKS; - IngestPipeline.taskScheduler.scheduleDataSourceIngestTask(this); + stage = IngestJobPipeline.Stages.FIRST_STAGE_ALL_TASKS; + IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); } } @@ -859,7 +852,7 @@ final class IngestPipeline { */ private void startSecondStage() { logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), job.getId())); //NON-NLS - stage = IngestPipeline.Stages.SECOND_STAGE; + stage = IngestJobPipeline.Stages.SECOND_STAGE; if (doUI) { startDataSourceIngestProgressBar(); } @@ -875,11 +868,11 @@ final class IngestPipeline { private void startArtifactIngestProgressBar() { if (doUI) { synchronized (resultsIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()); // RJCTODO + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.resultsIngest.displayName", this.dataSource.getName()); resultsIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { - IngestPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + IngestJobPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); return true; } }); @@ -908,12 +901,12 @@ final class IngestPipeline { // the user wants to cancel only the currently executing // data source ingest module or the entire ingest job. DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(IngestPipeline.this.getClass(), "IngestJob.cancellationDialog.title"); + String dialogTitle = NbBundle.getMessage(IngestJobPipeline.this.getClass(), "IngestJob.cancellationDialog.title"); JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); if (panel.cancelAllDataSourceIngestModules()) { - IngestPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + IngestJobPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); } else { - IngestPipeline.this.cancelCurrentDataSourceIngestModule(); + IngestJobPipeline.this.cancelCurrentDataSourceIngestModule(); } return true; } @@ -940,7 +933,7 @@ final class IngestPipeline { // the cancel button on the progress bar and the OK button // of a cancelation confirmation dialog supplied by // NetBeans. - IngestPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + IngestJobPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); return true; } }); @@ -955,53 +948,53 @@ final class IngestPipeline { * completed and does a stage transition if they are. */ private void checkForStageCompleted() { - if (ingestMode == IngestJob.Mode.BATCH) { - checkForStageCompletedBatch(); - } else { - checkForStageCompletedInStreamingMode(); - } - } - - /** - * Checks to see if the ingest tasks for the current stage of this job are - * completed and does a stage transition if they are. - */ - private void checkForStageCompletedBatch() { - // RJCTODO: This logic depends on carved/derived files for a task being - // added before the task is completed. See addFiles(). - synchronized (this.stageTransitionLock) { - if (IngestPipeline.taskScheduler.currentTasksAreCompleted(this)) { - switch (this.stage) { - case FIRST_STAGE_ALL_TASKS: - this.finishFirstStage(); - break; - case SECOND_STAGE: - this.shutDown(); - break; - } - } - } - } - - /** - * Checks to see if the ingest tasks for the current stage of this job are - * completed and does a stage transition if they are. - */ - private void checkForStageCompletedInStreamingMode() { - // RJCTODO: This logic depends on carved/derived files for a task being - // added before the task is completed. See addFiles(). - synchronized (this.stageTransitionLock) { - if (IngestPipeline.taskScheduler.currentTasksAreCompleted(this)) { - switch (this.stage) { + synchronized (stageTransitionLock) { + /* + * Note that there is nothing to do here for the + * FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY stage for streaming + * ingest, because we need to wait for the data source to be added + * to the job and the transition to FIRST_STAGE_ALL_TASKS. + */ + if (stage != Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY + && IngestJobPipeline.taskScheduler.currentTasksAreCompleted(this)) { + switch (stage) { case FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY: - // Nothing to do here - need to wait for the data source + /* + * Nothing to do here, need to wait for the data source + * to be added to the job and the transition to + * FIRST_STAGE_ALL_TASKS. + */ break; case FIRST_STAGE_ALL_TASKS: - // Finish file and data source ingest, start second stage (if applicable) - this.finishFirstStage(); + /* + * Tasks completed: + * + * 1. The first stage data source level ingest task (if + * any). + * + * 2. All file ingest tasks, including tasks for + * carved/derived files submitted by ingest modules in + * the first stage via IngestJobContext,addFilesToJob(). + * + * 3. The results tasks for results in the case database + * when the job started, plus any result tasks submitted + * by the ingest modules in other tasks in the first + * stage via IngestJobContext.addDataArtifactsToJob(). + */ + finishFirstStage(); break; case SECOND_STAGE: - this.shutDown(); + /* + * Tasks completed: + * + * 1. The second stage data source level ingest task (if + * any). + * + * 2. The results tasks for any results added by second + * stage ingest modules via + * IngestJobContext.addDataArtifactsToJob(). + */ + shutDown(); break; } } @@ -1015,45 +1008,16 @@ final class IngestPipeline { private void finishFirstStage() { logInfoMessage("Finished first stage analysis"); //NON-NLS - // Shut down the file ingest pipelines. Note that no shut down is - // required for the data source ingest pipeline because data source - // ingest modules do not have a shutdown() method. - List errors = new ArrayList<>(); - while (!this.fileIngestPipelinesQueue.isEmpty()) { + shutDownIngestTaskPipeline(currentDataSourceIngestPipeline); + while (!fileIngestPipelinesQueue.isEmpty()) { FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); - if (pipeline.isRunning()) { - errors.addAll(pipeline.shutDown()); - } - } - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); + shutDownIngestTaskPipeline(pipeline); } - if (this.doUI) { - // Finish the first stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } + finishProgressBar(dataSourceIngestProgress, dataSourceIngestProgressLock); + finishProgressBar(fileIngestProgress, fileIngestProgressLock); - // Finish the file ingest progress bar, if it hasn't already - // been finished. - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - this.fileIngestProgress.finish(); - this.fileIngestProgress = null; - } - } - } - - // RJCTODO: Shut down the data source level pipeline - /* - * Start the second stage, if appropriate. - */ - if (!this.cancelled && this.hasSecondStageDataSourceIngestModules()) { + if (!cancelled && hasSecondStageDataSourceIngestModules()) { startSecondStage(); } else { shutDown(); @@ -1065,22 +1029,15 @@ final class IngestPipeline { */ private void shutDown() { logInfoMessage("Finished all tasks"); //NON-NLS - stage = IngestPipeline.Stages.FINALIZATION; + stage = IngestJobPipeline.Stages.FINALIZATION; - // RJCTODO: SHould all the progress bars be attended to? - if (doUI) { - // Finish the second stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgress != null) { - dataSourceIngestProgress.finish(); - dataSourceIngestProgress = null; - } - } - } + shutDownIngestTaskPipeline(currentDataSourceIngestPipeline); + shutDownIngestTaskPipeline(artifactIngestPipeline); + + finishProgressBar(dataSourceIngestProgress, dataSourceIngestProgressLock); + finishProgressBar(fileIngestProgress, fileIngestProgressLock); + finishProgressBar(resultsIngestProgress, resultsIngestProgressLock); - // RJCTODO: Shut down pipelines - // RJCTODO: Put in a method if (ingestJobInfo != null) { if (cancelled) { try { @@ -1105,6 +1062,38 @@ final class IngestPipeline { job.notifyIngestPipelineShutDown(this); } + /** + * Shuts down an ingest task pipeline. + * + * @param pipeline The pipeline. + */ + private void shutDownIngestTaskPipeline(IngestTaskPipeline pipeline) { + if (pipeline.isRunning()) { + List errors = new ArrayList<>(); + errors.addAll(pipeline.shutDown()); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + } + + /** + * Finishes a progress bar. + * + * @param progress The progress bar. + * @param lock The lock that guards the progress bar. + */ + private void finishProgressBar(ProgressHandle progress, Object lock) { + if (doUI) { + synchronized (lock) { + if (progress != null) { + progress.finish(); + progress = null; + } + } + } + } + /** * Executes an ingest task by passing the task to the appropriate ingest * task pipeline. @@ -1138,7 +1127,7 @@ final class IngestPipeline { synchronized (dataSourceIngestPipelineLock) { if (!isCancelled() && !currentDataSourceIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); - errors.addAll(currentDataSourceIngestPipeline.performTask(task)); + errors.addAll(currentDataSourceIngestPipeline.executeTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } @@ -1203,7 +1192,7 @@ final class IngestPipeline { } else { fileIngestProgress.progress(file.getName(), (int) estimatedFilesToProcess); } - filesInProgress.add(file.getName()); // RJCTODO: Snapshot data structure only updated for UI case, I think this is wrong + filesInProgress.add(file.getName()); } } @@ -1211,7 +1200,7 @@ final class IngestPipeline { * Run the file through the modules in the pipeline. */ List errors = new ArrayList<>(); - errors.addAll(pipeline.performTask(task)); + errors.addAll(pipeline.executeTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors, file); } @@ -1222,7 +1211,7 @@ final class IngestPipeline { * Update the file ingest progress bar again, in * case the file was being displayed. */ - filesInProgress.remove(file.getName()); // RJCTODO: Snapshot data structure only updated for UI case, I think this is wrong + filesInProgress.remove(file.getName()); if (filesInProgress.size() > 0) { fileIngestProgress.progress(filesInProgress.get(0)); } else { @@ -1235,6 +1224,7 @@ final class IngestPipeline { } } catch (InterruptedException ex) { // RJCTODO This probablly should be logged, interrupt during wait for pipeline copy + // Also need to reset the flag... } finally { taskScheduler.notifyTaskCompleted(task); checkForStageCompleted(); @@ -1251,13 +1241,13 @@ final class IngestPipeline { try { if (!isCancelled() && !artifactIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); - errors.addAll(artifactIngestPipeline.performTask(task)); + errors.addAll(artifactIngestPipeline.executeTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } } } finally { - taskScheduler.notifyTaskCompleted(task); // RJCTODO: Looks like another overload is needed or a switch case + taskScheduler.notifyTaskCompleted(task); checkForStageCompleted(); } } @@ -1271,7 +1261,7 @@ final class IngestPipeline { void addStreamingIngestFiles(List fileObjIds) { if (hasFileIngestModules()) { if (stage.equals(Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY)) { - IngestPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); + IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } @@ -1324,7 +1314,7 @@ final class IngestPipeline { if (stage.equals(Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY) || stage.equals(Stages.FIRST_STAGE_ALL_TASKS) || stage.equals(Stages.SECOND_STAGE)) { - //taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); // RJCTODO + taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } @@ -1392,12 +1382,11 @@ final class IngestPipeline { * * @param workUnits Number of work units performed. */ - // RJCTODO: Is this used? void advanceDataSourceIngestProgressBar(int workUnits) { - if (this.doUI && !this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress("", workUnits); + if (doUI && !cancelled) { + synchronized (dataSourceIngestProgressLock) { + if (null != dataSourceIngestProgress) { + dataSourceIngestProgress.progress("", workUnits); } } } @@ -1410,10 +1399,10 @@ final class IngestPipeline { * @param currentTask The task name. */ void advanceDataSourceIngestProgressBar(String currentTask) { - if (this.doUI && !this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress(currentTask); + if (doUI && !cancelled) { + synchronized (dataSourceIngestProgressLock) { + if (null != dataSourceIngestProgress) { + dataSourceIngestProgress.progress(currentTask); } } } @@ -1502,7 +1491,7 @@ final class IngestPipeline { void cancel(IngestJob.CancellationReason reason) { this.cancelled = true; this.cancellationReason = reason; - IngestPipeline.taskScheduler.cancelPendingFileTasksForIngestJob(this); + IngestJobPipeline.taskScheduler.cancelPendingFileTasksForIngestJob(this); if (this.doUI) { synchronized (this.dataSourceIngestProgressLock) { @@ -1524,17 +1513,6 @@ final class IngestPipeline { checkForStageCompleted(); } - /** - * Set the current module name being run and the file name it is running on. - * To be used for more detailed cancelling. - * - * @param moduleName Name of module currently running. - * @param taskName Name of file the module is running on. - */ - void setCurrentFileIngestModule(String moduleName, String taskName) { - currentFileIngestTask = taskName; //RJCTODO: Is this correct? - } - /** * Queries whether or not cancellation, i.e., a shutdown of the data source * level and file level ingest pipelines for this job, has been requested. @@ -1633,9 +1611,6 @@ final class IngestPipeline { } } - /* - * RJCTODO - */ long processedFilesCount = 0; long estimatedFilesToProcessCount = 0; long snapShotTime = new Date().getTime(); @@ -1646,7 +1621,7 @@ final class IngestPipeline { estimatedFilesToProcessCount = this.estimatedFilesToProcess; snapShotTime = new Date().getTime(); } - tasksSnapshot = IngestPipeline.taskScheduler.getTasksSnapshotForJob(pipelineId); + tasksSnapshot = IngestJobPipeline.taskScheduler.getTasksSnapshotForJob(pipelineId); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 210392e16f..fe295d8539 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -127,7 +127,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { private final Map ingestJobsById = new HashMap<>(); private final ExecutorService dataSourceLevelIngestJobTasksExecutor; private final ExecutorService fileLevelIngestJobTasksExecutor; - private final ExecutorService dataArtifactIngestTasksExecutor; + private final ExecutorService resultIngestTasksExecutor; private final ExecutorService eventPublishingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-ingest-events-%d").build()); //NON-NLS; private final IngestMonitor ingestMonitor = new IngestMonitor(); private final ServicesMonitor servicesMonitor = ServicesMonitor.getInstance(); @@ -183,10 +183,10 @@ public class IngestManager implements IngestProgressSnapshotProvider { ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } - dataArtifactIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-artifact-ingest-%d").build()); //NON-NLS; + resultIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-results-ingest-%d").build()); //NON-NLS; threadId = nextIngestManagerTaskId.incrementAndGet(); + dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getResultIngestTaskQueue())); // RJCTODO - // dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); // ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); // RJCTODO: Where is the shut down code? } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java index 01e771742a..ec83a129c1 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java @@ -67,14 +67,14 @@ public interface IngestModule { */ default void startUp(IngestJobContext context) throws IngestModuleException { } - + /** * Invoked by Autopsy when an ingest job is completed (either because the * data has been analyzed or because the job was cancelled), before the * ingest module instance is discarded. The module should respond by doing * things like releasing private resources, submitting final results, and * posting a final ingest message. - * + * * IMPORTANT: If the module instances must share resources, the modules are * responsible for synchronizing access to the shared resources and doing * reference counting as required to release those resources correctly. @@ -87,7 +87,8 @@ public interface IngestModule { } /** - * An exception for the use of ingest modules. + * An exception for ingest modules to throw if they experience a start up + * error. */ public class IngestModuleException extends Exception { @@ -108,7 +109,7 @@ public interface IngestModule { } /** - * A return code for subclass process() methods. + * A return code for process() method implementations. */ public enum ProcessResult { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java index b0ac69756d..e473086d18 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java @@ -91,10 +91,7 @@ public interface IngestModuleFactory { * family of ingest modules the factory creates. For example, the Autopsy * core hash lookup ingest module factory provides a global settings panel * to import and create hash databases. The hash databases are then enabled - * or disabled per ingest job using an ingest job settings panel. If the - * module family does not have global settings, the factory may extend - * IngestModuleFactoryAdapter to get an implementation of this method that - * returns false. + * or disabled per ingest job using an ingest job settings panel. * * @return True if the factory provides a global settings panel. */ @@ -108,10 +105,7 @@ public interface IngestModuleFactory { * creates. For example, the Autopsy core hash lookup ingest module factory * provides a global settings panel to import and create hash databases. The * imported hash databases are then enabled or disabled per ingest job using - * ingest an ingest job settings panel. If the module family does not have a - * global settings, the factory may extend IngestModuleFactoryAdapter to get - * an implementation of this method that throws an - * UnsupportedOperationException. + * ingest an ingest job settings panel. * * @return A global settings panel. */ @@ -124,10 +118,7 @@ public interface IngestModuleFactory { * ingest modules the factory creates. For example, the Autopsy core hash * lookup ingest modules family uses hash databases imported or created * using its global settings panel. All of the hash databases are enabled by - * default for an ingest job. If the module family does not have per ingest - * job settings, the factory may extend IngestModuleFactoryAdapter to get an - * implementation of this method that returns an instance of the - * NoIngestModuleJobSettings class. + * default for an ingest job. * * @return The default ingest job settings. */ @@ -140,10 +131,7 @@ public interface IngestModuleFactory { * allow a user to make per ingest job settings for instances of the family * of ingest modules the factory creates. For example, the Autopsy core hash * lookup ingest module factory provides an ingest job settings panels to - * enable or disable hash databases per ingest job. If the module family - * does not have per ingest job settings, the factory may extend - * IngestModuleFactoryAdapter to get an implementation of this method that - * returns false. + * enable or disable hash databases per ingest job. * * @return True if the factory provides ingest job settings panels. */ @@ -156,10 +144,7 @@ public interface IngestModuleFactory { * settings for instances of the family of ingest modules the factory * creates. For example, the core hash lookup ingest module factory provides * an ingest job settings panel to enable or disable hash databases per - * ingest job. If the module family does not have per ingest job settings, - * the factory may extend IngestModuleFactoryAdapter to get an - * implementation of this method that throws an - * UnsupportedOperationException. + * ingest job. * * @param settings Per ingest job settings to initialize the panel. * @@ -171,9 +156,7 @@ public interface IngestModuleFactory { /** * Queries the factory to determine if it is capable of creating data source - * ingest modules. If the module family does not include data source ingest - * modules, the factory may extend IngestModuleFactoryAdapter to get an - * implementation of this method that returns false. + * ingest modules. * * @return True if the factory can create data source ingest modules. */ @@ -201,12 +184,8 @@ public interface IngestModuleFactory { * correctly. Also, more than one ingest job may be in progress at any given * time. This must also be taken into consideration when sharing resources * between module instances. modules. - *

- * If the module family does not include data source ingest modules, the - * factory may extend IngestModuleFactoryAdapter to get an implementation of - * this method that throws an UnsupportedOperationException. * - * @param settings The settings for the ingest job. + * @param ingestOptions The settings for the ingest job. * * @return A data source ingest module instance. */ @@ -216,9 +195,7 @@ public interface IngestModuleFactory { /** * Queries the factory to determine if it is capable of creating file ingest - * modules. If the module family does not include file ingest modules, the - * factory may extend IngestModuleFactoryAdapter to get an implementation of - * this method that returns false. + * modules. * * @return True if the factory can create file ingest modules. */ @@ -246,38 +223,27 @@ public interface IngestModuleFactory { * correctly. Also, more than one ingest job may be in progress at any given * time. This must also be taken into consideration when sharing resources * between module instances. modules. - *

- * If the module family does not include file ingest modules, the factory - * may extend IngestModuleFactoryAdapter to get an implementation of this - * method that throws an UnsupportedOperationException. * * @param settings The settings for the ingest job. * * @return A file ingest module instance. - * RJCTODO: Remove refs to adapters */ default FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { throw new UnsupportedOperationException(); } /** - * RJCTODO + * Queries the factory to determine if it is capable of creating data + * artifact ingest modules. * - * Queries the factory to determine if it is capable of creating file ingest - * modules. If the module family does not include file ingest modules, the - * factory may extend IngestModuleFactoryAdapter to get an implementation of - * this method that returns false. - * - * @return True if the factory can create file ingest modules. + * @return True or false. */ default boolean isDataArtifactIngestModuleFactory() { return false; } /** - * RJCTODO - * - * Creates a file ingest module instance. + * Creates a data artifact ingest module instance. *

* Autopsy will generally use the factory to several instances of each type * of module for each ingest job it performs. Completing an ingest job @@ -296,10 +262,6 @@ public interface IngestModuleFactory { * correctly. Also, more than one ingest job may be in progress at any given * time. This must also be taken into consideration when sharing resources * between module instances. modules. - *

- * If the module family does not include file ingest modules, the factory - * may extend IngestModuleFactoryAdapter to get an implementation of this - * method that throws an UnsupportedOperationException. * * @param settings The settings for the ingest job. * diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java index f376378235..c4d01e8bce 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java @@ -24,7 +24,7 @@ package org.sleuthkit.autopsy.ingest; * * NOTE: As of Java 8, interfaces can have default methods. IngestModuleFactory * now provides default no-op versions of all of its optional methods. This - * class is no longer needed and can be deprecated when convenient. + * class is no longer needed and can be DEPRECATED when convenient. */ public abstract class IngestModuleFactoryAdapter implements IngestModuleFactory { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index fde3449b75..7dbed1ffbf 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -29,7 +29,7 @@ import org.sleuthkit.datamodel.Content; abstract class IngestTask { private final static long NOT_SET = Long.MIN_VALUE; - private final IngestPipeline ingestJobPipeline; + private final IngestJobPipeline ingestJobPipeline; private long threadId; /** @@ -41,7 +41,7 @@ abstract class IngestTask { * @param ingestJobPipeline The ingest job pipeline to use to execute the * task. */ - IngestTask(IngestPipeline ingestJobPipeline) { + IngestTask(IngestJobPipeline ingestJobPipeline) { this.ingestJobPipeline = ingestJobPipeline; threadId = NOT_SET; } @@ -51,7 +51,7 @@ abstract class IngestTask { * * @return The ingest job pipeline. */ - IngestPipeline getIngestPipeline() { + IngestJobPipeline getIngestPipeline() { return ingestJobPipeline; } @@ -78,12 +78,8 @@ abstract class IngestTask { * using the ingest job pipeline specified when the task was created. * * @param threadId The numeric ID of the ingest thread executing this task. - * - * @throws InterruptedException This exception is thrown if the calling - * thread is interrupted while blocked waiting - * for some task operation to complete. - */ - void execute(long threadId) throws InterruptedException { // RJCTODO: Why does htis block? += */ + void execute(long threadId) { this.threadId = threadId; ingestJobPipeline.execute(this); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java index 1662d890df..f97f7cb1ff 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java @@ -26,18 +26,18 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; /** - * An abstract superclass for pipelines of ingest modules for a given ingest - * task type. Some examples of ingest task types: data source level ingest - * tasks, file ingest tasks, data artifact ingest tasks, etc. Subclasses need to - * implement a specialization of the inner PipelineModule abstract superclass - * for the type of ingest modules that make up the pipeline. + * A generic abstract superclass for pipelines of ingest modules for executing + * ingest tasks for an ingest job. The type parameter of the generic is an + * ingest task type. Some examples of ingest task types: data source level + * ingest tasks, file ingest tasks, data artifact ingest tasks, etc. Subclasses + * need to implement a specialization of the inner PipelineModule generic + * abstract superclass for the type of ingest modules that make up the pipeline. * * @param The ingest task type. */ abstract class IngestTaskPipeline { - private static final IngestManager ingestManager = IngestManager.getInstance(); - private final IngestPipeline ingestJobPipeline; + private final IngestJobPipeline ingestPipeline; private final List moduleTemplates; private final List> modules; private volatile Date startTime; @@ -45,25 +45,28 @@ abstract class IngestTaskPipeline { private volatile PipelineModule currentModule; /** - * Constructs an instance of an abstract superclass for pipelines of ingest - * modules for a given ingest task type. Some examples of ingest task types: - * data source level ingest tasks, file ingest tasks, data artifact ingest - * tasks, etc. Subclasses need to implement a specialization of the inner - * PipelineModule abstract superclass for the type of ingest modules that - * make up the pipeline. + * Constructs a generic abstract superclass for pipelines of ingest modules + * for executing ingest tasks for an ingest job. The type parameter of the + * generic is an ingest task type. Some examples of ingest task types: data + * source level ingest tasks, file ingest tasks, data artifact ingest tasks, + * etc. Subclasses need to implement a specialization of the inner + * PipelineModule generic abstract superclass for the type of ingest modules + * that make up the pipeline. * - * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. - * @param moduleTemplates The ingest module templates that define this - * pipeline. + * @param ingestPipeline The parent ingest pipeline for this ingest task + * pipeline. + * @param moduleTemplates The ingest module templates that define this + * ingest task pipeline. */ - IngestTaskPipeline(IngestPipeline ingestJobPipeline, List moduleTemplates) { - this.ingestJobPipeline = ingestJobPipeline; + IngestTaskPipeline(IngestJobPipeline ingestPipeline, List moduleTemplates) { + this.ingestPipeline = ingestPipeline; this.moduleTemplates = moduleTemplates; modules = new ArrayList<>(); } /** - * Indicates whether or not there are any ingest modules in this pipeline. + * Indicates whether or not there are any ingest modules in this ingest task + * pipeline. * * @return True or false. */ @@ -72,8 +75,8 @@ abstract class IngestTaskPipeline { } /** - * Queries whether or not this pipeline is running, i.e., started and not - * shut down. + * Queries whether or not this ingest task pipeline is running, i.e., + * started up and not yet shut down. * * @return True or false. */ @@ -82,9 +85,10 @@ abstract class IngestTaskPipeline { } /** - * Starts up the ingest modules in this pipeline. + * Starts up this ingest task pipeline by starting the ingest modules in the + * pipeline. * - * @return A list of ingest module startup errors, possibly empty. + * @return A list of ingest module start up errors, possibly empty. */ List startUp() { createIngestModules(moduleTemplates); @@ -92,7 +96,8 @@ abstract class IngestTaskPipeline { } /** - * Creates the ingest modules for this pipeline. + * Creates the ingest modules for this ingest task pipeline from ingest + * module templates. * * @param moduleTemplates The ingest module templates avaialble to this * pipeline. @@ -108,21 +113,21 @@ abstract class IngestTaskPipeline { /** * Determines if the type of ingest module that can be created from a given - * ingest module template should be added to this pipeline. If so, the - * ingest module is created and returned. + * ingest module template should be added to this ingest task pipeline. If + * so, the ingest module is created and returned. * - * @param ingestModuleTemplate The ingest module template to be used or - * ignored, as appropriate to the pipeline type. + * @param template The ingest module template to be used or ignored, as + * appropriate to the pipeline type. * * @return An Optional that is either empty or contains a newly created and * wrapped ingest module. */ - abstract Optional> acceptModuleTemplate(IngestModuleTemplate ingestModuleTemplate); + abstract Optional> acceptModuleTemplate(IngestModuleTemplate template); /** - * Starts up the ingest modules in the pipeline. + * Starts up the ingest modules in this ingest task pipeline. * - * @return A list of ingest module startup errors, possibly empty. + * @return A list of ingest module start up errors, possibly empty. */ private List startUpIngestModules() { startTime = new Date(); @@ -130,7 +135,7 @@ abstract class IngestTaskPipeline { List errors = new ArrayList<>(); for (PipelineModule module : modules) { try { - module.startUp(new IngestJobContext(ingestJobPipeline)); + module.startUp(new IngestJobContext(ingestPipeline)); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } @@ -139,7 +144,7 @@ abstract class IngestTaskPipeline { } /** - * Returns the start up time of this pipeline. + * Returns the start up time of this ingest task pipeline. * * @return The file processing start time, may be null if this pipeline has * not been started yet. @@ -159,20 +164,21 @@ abstract class IngestTaskPipeline { * @throws IngestTaskPipelineException Thrown if there is an error preparing * to perform the task. */ - abstract void prepareTask(T task) throws IngestTaskPipelineException; + abstract void prepareForTask(T task) throws IngestTaskPipelineException; /** - * Performs an ingest task using the ingest modules in this pipeline. + * Executes an ingest task using the ingest modules in this ingest task + * pipeline. * * @param task The task. * * @return A list of ingest module task processing errors, possibly empty. */ - List performTask(T task) { + List executeTask(T task) { List errors = new ArrayList<>(); - if (!this.ingestJobPipeline.isCancelled()) { + if (!this.ingestPipeline.isCancelled()) { try { - prepareTask(task); + prepareForTask(task); } catch (IngestTaskPipelineException ex) { errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS return errors; @@ -181,17 +187,17 @@ abstract class IngestTaskPipeline { try { currentModule = module; currentModule.setProcessingStartTime(); - module.performTask(ingestJobPipeline, task); + module.executeTask(ingestPipeline, task); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } - if (ingestJobPipeline.isCancelled()) { + if (ingestPipeline.isCancelled()) { break; } } } try { - completeTask(task); + cleanUpAfterTask(task); } catch (IngestTaskPipelineException ex) { errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS } @@ -200,7 +206,7 @@ abstract class IngestTaskPipeline { } /** - * Gets the currently running module. + * Gets the currently running ingest module. * * @return The module, possibly null if no module is currently running. */ @@ -216,10 +222,10 @@ abstract class IngestTaskPipeline { * @throws IngestTaskPipelineException Thrown if there is an error cleaning * up after performing the task. */ - abstract void completeTask(T task) throws IngestTaskPipelineException; + abstract void cleanUpAfterTask(T task) throws IngestTaskPipelineException; /** - * Shuts down all of the modules in the pipeline. + * Shuts down all of the ingest modules in this ingest task pipeline. * * @return A list of shut down errors, possibly empty. */ @@ -313,21 +319,22 @@ abstract class IngestTaskPipeline { } /** - * Performs an ingest task. + * Executes an ingest task using the process() method of the wrapped + * module. * - * @param ingestJobPipeline The ingest job pipeline that owns the ingest - * module pipeline this module belongs to. + * @param ingestJobPipeline The ingest pipeline that owns the ingest + * task pipeline this module belongs to. * @param task The task to process. * - * @throws IngestModuleException Excepton thrown if there is an error + * @throws IngestModuleException Exception thrown if there is an error * performing the task. */ - abstract void performTask(IngestPipeline ingestJobPipeline, T task) throws IngestModuleException; - + abstract void executeTask(IngestJobPipeline ingestJobPipeline, T task) throws IngestModuleException; + @Override public void shutDown() { module.shutDown(); - } + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 090b13a053..7c5ab75a69 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -66,7 +66,7 @@ final class IngestTasksScheduler { @GuardedBy("this") private final Queue streamedFileIngestTasksQueue; private final IngestTaskTrackingQueue fileIngestTasksQueue; - private final IngestTaskTrackingQueue artifactIngestTasksQueue; + private final IngestTaskTrackingQueue resultIngestTasksQueue; /** * Gets the ingest tasks scheduler singleton that creates ingest tasks for @@ -91,7 +91,7 @@ final class IngestTasksScheduler { batchedFileIngestTasksQueue = new LinkedList<>(); fileIngestTasksQueue = new IngestTaskTrackingQueue(); streamedFileIngestTasksQueue = new LinkedList<>(); - artifactIngestTasksQueue = new IngestTaskTrackingQueue(); + resultIngestTasksQueue = new IngestTaskTrackingQueue(); } /** @@ -120,8 +120,8 @@ final class IngestTasksScheduler { * * @return The queue. */ - BlockingIngestTaskQueue getDataArtifactIngestTaskQueue() { - return artifactIngestTasksQueue; + BlockingIngestTaskQueue getResultIngestTaskQueue() { + return resultIngestTasksQueue; } /** @@ -149,7 +149,7 @@ final class IngestTasksScheduler { * target Content of the task to the pipeline for * processing by the pipeline's ingest modules. */ - synchronized void scheduleDataSourceAndFileIngestTasks(IngestPipeline ingestPipeline) { + synchronized void scheduleDataSourceAndFileIngestTasks(IngestJobPipeline ingestPipeline) { if (!ingestPipeline.isCancelled()) { scheduleDataSourceIngestTask(ingestPipeline); scheduleFileIngestTasks(ingestPipeline, Collections.emptyList()); @@ -178,7 +178,7 @@ final class IngestTasksScheduler { * target Content of the task to the pipeline for * processing by the pipeline's ingest modules. */ - synchronized void scheduleDataSourceIngestTask(IngestPipeline ingestPipeline) { + synchronized void scheduleDataSourceIngestTask(IngestJobPipeline ingestPipeline) { if (!ingestPipeline.isCancelled()) { DataSourceIngestTask task = new DataSourceIngestTask(ingestPipeline); try { @@ -217,7 +217,7 @@ final class IngestTasksScheduler { * empty, then all if the files from the data source * are candidates for scheduling. */ - synchronized void scheduleFileIngestTasks(IngestPipeline ingestPipeline, Collection files) { + synchronized void scheduleFileIngestTasks(IngestJobPipeline ingestPipeline, Collection files) { if (!ingestPipeline.isCancelled()) { Collection candidateFiles; if (files.isEmpty()) { @@ -260,7 +260,7 @@ final class IngestTasksScheduler { * processing by the pipeline's ingest modules. * @param files A list of file object IDs for the streamed files. */ - synchronized void scheduleStreamedFileIngestTasks(IngestPipeline ingestPipeline, List fileIds) { + synchronized void scheduleStreamedFileIngestTasks(IngestJobPipeline ingestPipeline, List fileIds) { if (!ingestPipeline.isCancelled()) { for (long id : fileIds) { /* @@ -303,7 +303,7 @@ final class IngestTasksScheduler { * processing by the pipeline's ingest modules. * @param files The files. */ - synchronized void fastTrackFileIngestTasks(IngestPipeline ingestPipeline, Collection files) { + synchronized void fastTrackFileIngestTasks(IngestJobPipeline ingestPipeline, Collection files) { if (!ingestPipeline.isCancelled()) { /* * Put the files directly into the queue for the file ingest @@ -351,7 +351,7 @@ final class IngestTasksScheduler { * target Content of the task to the pipeline for * processing by the pipeline's ingest modules. */ - synchronized void scheduleDataArtifactIngestTasks(IngestPipeline ingestPipeline) { + synchronized void scheduleDataArtifactIngestTasks(IngestJobPipeline ingestPipeline) { if (!ingestPipeline.isCancelled()) { Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); try { @@ -388,12 +388,12 @@ final class IngestTasksScheduler { * processing by the pipeline's ingest modules. * @param artifacts The artifacts. */ - synchronized void scheduleDataArtifactIngestTasks(IngestPipeline ingestPipeline, List artifacts) { // RJCTODO: WHy are cancellation checks in the scheduler instead of in the pipeline? + synchronized void scheduleDataArtifactIngestTasks(IngestJobPipeline ingestPipeline, List artifacts) { if (!ingestPipeline.isCancelled()) { for (DataArtifact artifact : artifacts) { DataArtifactIngestTask task = new DataArtifactIngestTask(ingestPipeline, artifact); try { - this.artifactIngestTasksQueue.putLast(task); + this.resultIngestTasksQueue.putLast(task); } catch (InterruptedException ex) { DataSource dataSource = ingestPipeline.getDataSource(); logger.log(Level.WARNING, String.format("Interrupted while enqueuing data artifact tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS @@ -432,7 +432,7 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(DataArtifactIngestTask task) { - artifactIngestTasksQueue.taskCompleted(task); + resultIngestTasksQueue.taskCompleted(task); } /** @@ -443,14 +443,14 @@ final class IngestTasksScheduler { * * @return True or false. */ - synchronized boolean currentTasksAreCompleted(IngestPipeline ingestPipeline) { + synchronized boolean currentTasksAreCompleted(IngestJobPipeline ingestPipeline) { long pipelineId = ingestPipeline.getId(); return !(dataSourceIngestTasksQueue.hasTasksForJob(pipelineId) || hasTasksForJob(topLevelFileIngestTasksQueue, pipelineId) || hasTasksForJob(batchedFileIngestTasksQueue, pipelineId) || hasTasksForJob(streamedFileIngestTasksQueue, pipelineId) || fileIngestTasksQueue.hasTasksForJob(pipelineId) - || artifactIngestTasksQueue.hasTasksForJob(pipelineId)); + || resultIngestTasksQueue.hasTasksForJob(pipelineId)); } /** @@ -472,7 +472,7 @@ final class IngestTasksScheduler { * * @param ingestJobPipeline The ingest pipeline for the job. */ - synchronized void cancelPendingFileTasksForIngestJob(IngestPipeline ingestJobPipeline) { + synchronized void cancelPendingFileTasksForIngestJob(IngestJobPipeline ingestJobPipeline) { long jobId = ingestJobPipeline.getId(); removeTasksForJob(topLevelFileIngestTasksQueue, jobId); removeTasksForJob(batchedFileIngestTasksQueue, jobId); @@ -512,7 +512,6 @@ final class IngestTasksScheduler { } } } catch (TskCoreException ex) { - // RJCTODO: Improve logging logger.log(Level.SEVERE, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS } } @@ -791,7 +790,7 @@ final class IngestTasksScheduler { Iterator iterator = tasks.iterator(); while (iterator.hasNext()) { IngestTask task = iterator.next(); - if (task.getIngestPipeline().getId() == pipelineId) { // RJCTODO: Is this right? Why is there no synchronization, are these collections thread safe? + if (task.getIngestPipeline().getId() == pipelineId) { iterator.remove(); } } @@ -817,7 +816,6 @@ final class IngestTasksScheduler { } /** - * //RJCTODO * Returns a snapshot of the states of the tasks in progress for an ingest * job. * diff --git a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java index 3dbe2c7360..b10d2cd793 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2020 Basis Technology Corp. + * Copyright 2020-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,6 +33,7 @@ import java.util.Collection; import java.util.Date; import java.util.Set; import java.util.HashSet; +import java.util.List; import java.util.TimeZone; import java.util.logging.Level; import org.apache.commons.lang3.StringUtils; @@ -56,17 +57,20 @@ import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.modules.pictureanalyzer.spi.PictureProcessor; +import org.sleuthkit.datamodel.AnalysisResult; +import org.sleuthkit.datamodel.DataArtifact; /** * Extracts EXIF metadata from JPEG, TIFF, and WAV files. Currently only date, * latitude, longitude, altitude, device model, and device make are extracted. - * + * * User content suspected artifacts are also created by this processor. */ @ServiceProvider(service = PictureProcessor.class) public class EXIFProcessor implements PictureProcessor { private static final Logger logger = Logger.getLogger(EXIFProcessor.class.getName()); + private static final BlackboardArtifact.Type EXIF_METADATA = new BlackboardArtifact.Type(TSK_METADATA_EXIF); @Override @NbBundle.Messages({ @@ -143,25 +147,43 @@ public class EXIFProcessor implements PictureProcessor { if (context.fileIngestIsCancelled()) { return; } - + final Blackboard blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard(); if (!attributes.isEmpty() && !blackboard.artifactExists(file, TSK_METADATA_EXIF, attributes)) { - - final BlackboardArtifact exifArtifact = file.newArtifact(TSK_METADATA_EXIF); + /* + * Create a TSK_METADATA_EXIF data artifact. + */ + final List newArtifacts = new ArrayList<>(); + final List newDataArtifacts = new ArrayList<>(); + final DataArtifact exifArtifact = file.newDataArtifact(EXIF_METADATA, attributes, null); + newArtifacts.add(exifArtifact); + newDataArtifacts.add(exifArtifact); + + /* + * Create a TSK_USER_CONTENT_SUSPECTED analysis result. + */ final BlackboardArtifact userSuspectedArtifact = file.newArtifact(TSK_USER_CONTENT_SUSPECTED); - exifArtifact.addAttributes(attributes); userSuspectedArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, Bundle.ExifProcessor_userContent_description())); + newArtifacts.add(userSuspectedArtifact); + + /* + * Post all of the artifacts to the blackboard. + */ try { - // index the artifact for keyword search - blackboard.postArtifact(exifArtifact, MODULE_NAME); - blackboard.postArtifact(userSuspectedArtifact, MODULE_NAME); + blackboard.postArtifacts(newArtifacts, MODULE_NAME); } catch (Blackboard.BlackboardException ex) { - logger.log(Level.SEVERE, "Unable to index blackboard artifact " + exifArtifact.getArtifactID(), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Error posting TSK_METADATA_EXIF and TSK_USER_CONTENT_SUSPECTED artifacts for %s (object ID = %d)", file.getName(), file.getId()), ex); //NON-NLS MessageNotifyUtil.Notify.error( Bundle.ExifProcessor_indexError_message(), exifArtifact.getDisplayName()); } + + /* + * Add the data artifact to the ingest job for processing by the + * data artifact ingest modules. + */ + context.addDataArtifactsToJob(newDataArtifacts); } } catch (TskCoreException ex) { logger.log(Level.WARNING, "Failed to create blackboard artifact for " //NON-NLS From 8e7db568532724df4ff6b251a952c4b74386a6f7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 15 Mar 2021 17:49:54 -0400 Subject: [PATCH 07/63] 7332 interim check in --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index f5354503ed..ab3a29c477 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -397,9 +397,8 @@ final class IngestJobPipeline { } } catch (InterruptedException ex) { /* - * RJCTODO: This is incorrect. If this thread is interrupted, the - * pipeline is incomplete and should not be used. Either fix this or - * write a story about it. + * RC: This is not incorrect. If this thread is interrupted, the + * pipeline is incomplete and should not be used. We are currently relying on */ Thread.currentThread().interrupt(); } From 51726730332f5c5c3b3b2b32cb601153ee61ab6d Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 27 May 2021 14:19:15 -0400 Subject: [PATCH 08/63] Atrifact ingest pipeline work --- .../CentralRepoDataArtifactIngestModule.java | 3 +- .../CentralRepoIngestModuleFactory.java | 44 +- .../autopsy/ingest/Bundle.properties | 2 +- .../autopsy/ingest/Bundle.properties-MERGED | 2 +- .../ingest/DataArtifactIngestPipeline.java | 16 +- .../ingest/DataSourceIngestPipeline.java | 8 +- .../autopsy/ingest/FileIngestPipeline.java | 8 +- .../autopsy/ingest/FileIngestTask.java | 6 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 11 +- .../autopsy/ingest/IngestJobPipeline.java | 794 +++++++++--------- .../autopsy/ingest/IngestManager.java | 47 +- .../autopsy/ingest/IngestModuleError.java | 2 +- .../sleuthkit/autopsy/ingest/IngestTask.java | 4 +- .../autopsy/ingest/IngestTaskPipeline.java | 249 +++--- .../autopsy/ingest/IngestTasksScheduler.java | 156 +--- 15 files changed, 660 insertions(+), 692 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java index 7ecaf9b46a..9820307ecd 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java @@ -43,8 +43,7 @@ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestMo IngestServices.getInstance().postMessage(IngestMessage.createMessage( IngestMessage.MessageType.INFO, CentralRepoIngestModuleFactory.getModuleName(), - "Data Artifacts Processed", //NON-NLS - String.format("Count = %d", artifactCounter.get()))); //NON-NLS + String.format("%d data artifacts processed", artifactCounter.get()))); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java index 7fbf162e83..39c80abefc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java @@ -31,7 +31,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; /** - * An ingest module factory for Central Repository ingest modules. + * Factory for Central Repository ingest modules. */ @ServiceProvider(service = org.sleuthkit.autopsy.ingest.IngestModuleFactory.class) @NbBundle.Messages({ @@ -41,7 +41,7 @@ import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { /** - * Gets the name of the module. + * Get the name of the module. * * @return The module name. */ @@ -64,6 +64,26 @@ public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { return Version.getVersion(); } + @Override + public boolean isFileIngestModuleFactory() { + return true; + } + + @Override + public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { + if (settings instanceof IngestSettings) { + return new CentralRepoIngestModule((IngestSettings) settings); + } + /* + * Earlier versions of the modules had no ingest job settings. Create a + * module with the default settings. + */ + if (settings instanceof NoIngestModuleIngestJobSettings) { + return new CentralRepoIngestModule((IngestSettings) getDefaultIngestJobSettings()); + } + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); + } + @Override public boolean hasGlobalSettingsPanel() { return true; @@ -101,26 +121,6 @@ public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); } - @Override - public boolean isFileIngestModuleFactory() { - return true; - } - - @Override - public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { - if (settings instanceof IngestSettings) { - return new CentralRepoIngestModule((IngestSettings) settings); - } - /* - * Earlier versions of the modules had no ingest job settings. Create a - * module with the default settings. - */ - if (settings instanceof NoIngestModuleIngestJobSettings) { - return new CentralRepoIngestModule((IngestSettings) getDefaultIngestJobSettings()); - } - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); - } - @Override public boolean isDataArtifactIngestModuleFactory() { return true; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index 9f00ead0b0..978e361baa 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -20,7 +20,7 @@ IngestMessagePanel.totalUniqueMessagesNameVal.text=- IngestJob.progress.dataSourceIngest.initialDisplayName=Analyzing {0} IngestJob.progress.dataSourceIngest.displayName={0} for {1} IngestJob.progress.fileIngest.displayName=Analyzing files from {0} -IngestJob.progress.resultsIngest.displayName=Analyzing results from {0} +IngestJob.progress.dataArtifactIngest.displayName=Analyzing data artifacts from {0} IngestJob.progress.cancelling=Cancelling... IngestJob.cancellationDialog.title=Cancel Ingest IngestDialog.startButton.title=Start diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED index f74a29a397..2226da6186 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED @@ -36,7 +36,7 @@ IngestMessagePanel.totalUniqueMessagesNameVal.text=- IngestJob.progress.dataSourceIngest.initialDisplayName=Analyzing {0} IngestJob.progress.dataSourceIngest.displayName={0} for {1} IngestJob.progress.fileIngest.displayName=Analyzing files from {0} -IngestJob.progress.resultsIngest.displayName=Analyzing results from {0} +IngestJob.progress.dataArtifactIngest.displayName=Analyzing data artifacts from {0} IngestJob.progress.cancelling=Cancelling... IngestJob.cancellationDialog.title=Cancel Ingest IngestDialog.startButton.title=Start diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java index b8b7c95cd3..5b6ca85120 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java @@ -20,27 +20,21 @@ package org.sleuthkit.autopsy.ingest; import java.util.List; import java.util.Optional; -import java.util.logging.Level; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataArtifact; -import org.sleuthkit.datamodel.TskCoreException; /** - * A pipeline of data artifact ingest modules for performing data artifact + * A pipeline of data artifact ingest modules for executing data artifact * ingest tasks for an ingest job. */ final class DataArtifactIngestPipeline extends IngestTaskPipeline { - private static final Logger logger = Logger.getLogger(IngestJobPipeline.class.getName()); - /** - * Constructs a pipeline of data artifact ingest modules for performing data + * Constructs a pipeline of data artifact ingest modules for executing data * artifact ingest tasks for an ingest job. * * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. * @param moduleTemplates The ingest module templates that define this - * pipeline. + * pipeline. May be an empty list. */ DataArtifactIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { super(ingestJobPipeline, moduleTemplates); @@ -65,7 +59,7 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline { @@ -73,7 +67,7 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline { } /** - * Adds a file to a file cache used to update the case database with new + * Adds a file to a file cache used to update the case database with any new * properties added to the files in the cache by the ingest modules that * processed them. If adding the file to the cache fills the cache, a batch * update is done immediately. @@ -203,11 +203,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { throw new IngestModuleException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS } ingestManager.setIngestTaskProgress(task, getDisplayName()); - ProcessResult result = module.process(file); - // See JIRA-7449 -// if (result == ProcessResult.ERROR) { -// throw new IngestModuleException(String.format("%s experienced an error analyzing %s (file objId = %d)", getDisplayName(), file.getName(), file.getId())); //NON-NLS -// } + module.process(file); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 48ae2d9820..6dfc0729d3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -95,8 +95,8 @@ final class FileIngestTask extends IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - IngestJobPipeline thisPipeline = getIngestPipeline(); - IngestJobPipeline otherPipeline = other.getIngestPipeline(); + IngestJobPipeline thisPipeline = getIngestJobPipeline(); + IngestJobPipeline otherPipeline = other.getIngestJobPipeline(); if (thisPipeline != otherPipeline && (thisPipeline == null || !thisPipeline.equals(otherPipeline))) { return false; } @@ -106,7 +106,7 @@ final class FileIngestTask extends IngestTask { @Override public int hashCode() { int hash = 5; - hash = 47 * hash + Objects.hashCode(getIngestPipeline()); + hash = 47 * hash + Objects.hashCode(getIngestJobPipeline()); hash = 47 * hash + Objects.hashCode(this.fileId); return hash; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index bc3c127a0a..5f0612bd4f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -138,6 +138,14 @@ public final class IngestJob { return this.id; } + /** + * RJCTODO + * @return + */ + List getFiles() { + return files; + } + /** * Checks to see if this ingest job has at least one non-empty ingest module * pipeline (first or second stage data-source-level pipeline or file-level @@ -183,8 +191,7 @@ public final class IngestJob { * * @return A collection of ingest module start up errors, empty on success. */ - List start() { - + List start() throws InterruptedException { /* * Set up the ingest job pipelines, one for each data source to be * ingested by this job. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index ab3a29c477..fd139e12d8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -31,6 +31,7 @@ import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; +import javax.annotation.concurrent.GuardedBy; import javax.swing.JOptionPane; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; @@ -55,97 +56,84 @@ import org.sleuthkit.datamodel.DataArtifact; import org.sleuthkit.datamodel.DataSource; /** - * An ingest pipeline that works with the ingest tasks scheduler to coordinate - * the creation, scheduling, and execution of ingest tasks for one of the data - * sources in an ingest job. An ingest job pipeline has one-to-many child ingest - * task pipelines. An ingest task pipeline is a sequence of ingest modules of a - * given type that have been enabled and configured as part of the settings for - * the ingest job. + * An ingest-job-level pipeline that works with the ingest tasks scheduler to + * coordinate the creation, scheduling, and execution of ingest tasks for one of + * the data sources in an ingest job. An ingest job pipeline is actually + * composed of multiple ingest task pipelines. Each ingest task pipeline is a + * sequence of ingest modules of a given type (e.g., data source level, file + * level, or artifact ingest modules) that have been enabled and configured as + * part of the ingest job settings. */ final class IngestJobPipeline { - /* - * The class names of the proxy classes Jython generates for Python classes - * look something like: - * "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" - */ - private static final Pattern JYTHON_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); private static final String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy"; private static final Logger logger = Logger.getLogger(IngestJobPipeline.class.getName()); + + /* + * A regular expression for identifying the proxy classes Jython generates + * for ingest module factories classes written using Python. For example: + * org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14 + */ + private static final Pattern JYTHON_MODULE_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); + + /* + * These fields define an ingest pipeline: the parent ingest job, a pipeline + * ID, the user's ingest job settings, and the data source to be analyzed. + * Optionally, there is a set of files to be analyzed, instead of analyzing + * ALL of the files in the data source. + * + * The pipeline ID is used to associate the pipeline with its ingest tasks. + * The ingest job ID cannot be used for this purpose because the parent + * ingest job may have more than one data source and each data source gets + * its own pipeline. + */ private final IngestJob job; + private static final AtomicLong nextPipelineId = new AtomicLong(0L); + private final long pipelineId; private final IngestJobSettings settings; private DataSource dataSource; private final List files; - private final long createTime; /* - * An ingest pipeline interacts with the ingest task scheduler to schedule - * initial ingest tasks, determine whether or not there are ingest tasks - * still to be executed, and to schedule additional tasks submitted by - * ingest modules. For example, a file carving module can add carved files - * to an ingest job and many modules will add data artifacts to an ingest - * job. - * - * The pipeline ID is used to associate the pipeline with its tasks. The - * ingest job ID cannot be used for this because an ingest job may have more - * than one pipeline (one pipeline per data source). - */ - private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - private static final AtomicLong nextPipelineId = new AtomicLong(0L); - private final long pipelineId; - - /* - * An ingest pipeline runs its child ingest task pipelines in stages. + * An ingest pipeline runs its ingest modules in stages. */ private static enum Stages { - /* - * The ingest pipeline is setting up its child ingest task pipelines. + * The pipeline is instantiating ingest modules and loading them into + * its ingest task pipelines. */ INITIALIZATION, /* - * The ingest pipeline is only running its file and result ingest task - * pipelines because the data source for the job has not been supplied - * to it yet. This stage is only used for streaming ingest. + * The pipeline is running file ingest modules on files streamed to it + * by a data source processor. The data source has not been added to the + * pipeline yet. */ - FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY, + FIRST_STAGE_STREAMING, /* - * The ingest pipeline is running its high priority data source level - * ingest task pipeline, its file ingest task pipelines, and its result - * ingest task pipelines. + * The pipeline is running one or more of the following three types of + * ingest modules: higher priority data source level ingest modules, + * file ingest modules, and artifact ingest modules. */ - FIRST_STAGE_ALL_TASKS, + FIRST_STAGE, /** - * The ingest pipeline is running its lower priority data source level - * ingest task pipeline and its result task pipelines. + * The pipeline is running lower priority, usually long-running, data + * source level ingest modules and artifact ingest modules. */ SECOND_STAGE, /** - * The ingest pipeline is shutting down its child ingest task pipelines. + * The pipeline is shutting down its ingest modules. */ FINALIZATION }; - private volatile Stages stage = IngestJobPipeline.Stages.INITIALIZATION; + @GuardedBy("stageTransitionLock") + private Stages stage = IngestJobPipeline.Stages.INITIALIZATION; private final Object stageTransitionLock = new Object(); - /* - * An ingest pipeline has at most a single data artifact ingest task - * pipeline. - * - * The pipeline may be empty, depending on which modules are enabled in the - * ingest job settings. - */ - private DataArtifactIngestPipeline artifactIngestPipeline; - /** * An ingest pipeline has separate data source level ingest task pipelines - * for the first and second processing stages. Longer running, lower - * priority modules belong in the second stage pipeline, although this - * cannot be enforced. - * - * Either or both pipelines may be empty, depending on which modules are - * enabled in the ingest job settings. + * for the first and second stages. Longer running, lower priority modules + * belong in the second stage pipeline. */ private final Object dataSourceIngestPipelineLock = new Object(); private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; @@ -154,93 +142,122 @@ final class IngestJobPipeline { /** * An ingest pipeline has a collection of identical file ingest task - * pipelines, one for each file ingest thread in the ingest manager. - * - * The ingest threads take and return task pipelines using a blocking queue. - * Additionally, a fixed list of all of the file pipelines is used to allow - * cycling through each of the individual task pipelines to check their - * status. - * - * The pipelines may be empty, depending on which modules are enabled in the - * ingest job settings. + * pipelines, one for each file ingest thread in the ingest manager. The + * ingest threads take ingest task pipelines as they need them and return + * the pipelines using a blocking queue. Additionally, a fixed list of all + * of the file pipelines is used to cycle through each of the individual + * task pipelines to check their status. */ private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); private final List fileIngestPipelines = new ArrayList<>(); + /* + * An ingest pipeline has a single artifact ingest task pipeline + */ + private DataArtifactIngestPipeline artifactIngestPipeline; + /** - * An ingest pipeline supports cancellation of either the currently running - * data source level ingest pipeline or all of its child pipelines. - * Cancellation works by setting flags that are checked by the ingest task - * pipelines every time they transition from from one module to another. - * Modules are also expected to check these flags and stop processing if - * they are set. This means that there can be a variable length delay - * between a cancellation request and its fulfillment. + * An ingest pipeline supports cancellation of just its currently running + * data source level ingest task pipeline or cancellation of ALL of its + * child ingest task pipelines. Cancellation works by setting flags that are + * checked by the ingest task pipelines every time they transition from one + * module to another. Modules are also expected to check these flags (via + * the ingest job context) and stop processing if they are set. This means + * that there can be a variable length delay between a cancellation request + * and its fulfillment. */ private volatile boolean currentDataSourceIngestModuleCancelled; private final List cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>(); private volatile boolean cancelled; private volatile IngestJob.CancellationReason cancellationReason = IngestJob.CancellationReason.NOT_CANCELLED; + /* + * An ingest pipeline interacts with the ingest task scheduler to create and + * queue ingest tasks and to determine whether or not there are ingest tasks + * still to be executed so that the pipeline can transition through its + * stages. The ingest modules in the pipeline can schedule ingest tasks as + * well (via the ingest job context). For example, a file carving module can + * add carved files to the ingest job and most modules will add data + * artifacts to the ingest job. + */ + private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); + /** * If running in a GUI, an ingest pipeline reports progress and allows a * user to cancel either an individual data source level ingest module or * all of its ingest tasks using progress bars in the lower right hand - * corner of the main application window. There is also support for ingest - * process snapshots and recording ingest job details in the case database. + * corner of the main application window. There is also support for taking + * ingest progress snapshots and for recording ingest job details in the + * case database. */ private final boolean doUI; - private final Object resultsIngestProgressLock = new Object(); - private ProgressHandle resultsIngestProgress; private final Object dataSourceIngestProgressLock = new Object(); - private ProgressHandle dataSourceIngestProgress; + private ProgressHandle dataSourceIngestProgressBar; private final Object fileIngestProgressLock = new Object(); private final List filesInProgress = new ArrayList<>(); private long estimatedFilesToProcess; private long processedFiles; - private ProgressHandle fileIngestProgress; + private ProgressHandle fileIngestProgressBar; + private final Object artifactIngestProgressLock = new Object(); + private ProgressHandle artifactIngestProgressBar; private volatile IngestJobInfo ingestJobInfo; /** - * Constructs an ingest pipeline that works with the ingest tasks scheduler - * to coordinate the creation, scheduling, and execution of ingest tasks for - * one of the data sources in an ingest job. An ingest job pipeline has - * one-to-many child ingest task pipelines. An ingest task pipeline is a - * sequence of ingest modules of a given type that have been enabled and - * configured as part of the settings for the ingest job. + * An ingest pipeline uses this field to report its creation time. + */ + private final long createTime; + + /** + * Constructs an ingest-job-level pipeline that works with the ingest tasks + * scheduler to coordinate the creation, scheduling, and execution of ingest + * tasks for one of the data sources in an ingest job. An ingest job + * pipeline is actually composed of multiple ingest task pipelines. Each + * ingest task pipeline is a sequence of ingest modules of a given type + * (e.g., data source level, file level, or artifact ingest modules) that + * have been enabled and configured as part of the ingest job settings. * * @param job The ingest job. - * @param dataSource The data source. + * @param dataSource One of the data sources that are the subjects of the + * ingest job. * @param settings The ingest settings for the ingest job. + * + * @throws InterruptedException Exception thrown if the thread in which the + * pipeline is being created is interrupted. */ - IngestJobPipeline(IngestJob job, Content dataSource, IngestJobSettings settings) { + IngestJobPipeline(IngestJob job, Content dataSource, IngestJobSettings settings) throws InterruptedException { this(job, dataSource, Collections.emptyList(), settings); } /** - * Constructs an ingest pipeline that works with the ingest tasks scheduler - * to coordinate the creation, scheduling, and execution of ingest tasks for - * one of the data sources in an ingest job. An ingest job pipeline has - * one-to-many child ingest task pipelines. An ingest task pipeline is a - * sequence of ingest modules of a given type that have been enabled and - * configured as part of the settings for the ingest job. + * Constructs an ingest-job-level pipeline that works with the ingest tasks + * scheduler to coordinate the creation, scheduling, and execution of ingest + * tasks for one of the data sources in an ingest job. An ingest job + * pipeline is actually composed of multiple ingest task pipelines. Each + * ingest task pipeline is a sequence of ingest modules of a given type + * (e.g., data source level, file level, or artifact ingest modules) that + * have been enabled and configured as part of the ingest job settings. * * @param job The ingest job. - * @param dataSource The data source for the ingest job. - * @param files The subset of the files for the data source. If the - * list is empty, all of the files in the data source are - * processed. + * @param dataSource One of the data sources that are the subjects of the + * ingest job. + * @param files A subset of the files from the data source. If the list + * is empty, ALL of the files in the data source are an + * analyzed. * @param settings The ingest settings for the ingest job. + * + * @throws InterruptedException Exception thrown if the thread in which the + * pipeline is being created is interrupted. */ - IngestJobPipeline(IngestJob job, Content dataSource, List files, IngestJobSettings settings) { + IngestJobPipeline(IngestJob job, Content dataSource, List files, IngestJobSettings settings) throws InterruptedException { if (!(dataSource instanceof DataSource)) { throw new IllegalArgumentException("Passed dataSource that does not implement the DataSource interface"); //NON-NLS } this.job = job; - this.settings = settings; + pipelineId = IngestJobPipeline.nextPipelineId.getAndIncrement(); this.dataSource = (DataSource) dataSource; this.files = new ArrayList<>(); this.files.addAll(files); - pipelineId = IngestJobPipeline.nextPipelineId.getAndIncrement(); + this.settings = settings; doUI = RuntimeProperties.runningWithGUI(); createTime = new Date().getTime(); stage = Stages.INITIALIZATION; @@ -278,12 +295,12 @@ final class IngestJobPipeline { * will be parsed to return * "GPX_Parser_Module.GPXParserFileIngestModuleFactory." * - * @param className The class name. + * @param className The canonical class name. * - * @return The jython name or null if not in jython package. + * @return The Jython proxu class name or null if the extraction fails. */ private static String getModuleNameFromJythonClassName(String className) { - Matcher m = JYTHON_REGEX.matcher(className); + Matcher m = JYTHON_MODULE_REGEX.matcher(className); if (m.find()) { return String.format("%s.%s", m.group(1), m.group(2)); //NON-NLS } else { @@ -301,7 +318,7 @@ final class IngestJobPipeline { * @param jythonMapping Mapping for Jython ingest module templates. * @param template The ingest module template. */ - private static void addIngestModuleTermplateToMaps(Map mapping, Map jythonMapping, IngestModuleTemplate template) { + private static void addIngestModuleTemplateToMaps(Map mapping, Map jythonMapping, IngestModuleTemplate template) { String className = template.getModuleFactory().getClass().getCanonicalName(); String jythonName = getModuleNameFromJythonClassName(className); if (jythonName != null) { @@ -313,31 +330,33 @@ final class IngestJobPipeline { /** * Creates the child ingest task pipelines for this ingest pipeline. + * + * @throws InterruptedException Exception thrown if the thread in which the + * task pipelines are being created is + * interrupted. */ - private void createIngestTaskPipelines() { + private void createIngestTaskPipelines() throws InterruptedException { /* * Get the enabled ingest module templates from the ingest job settings. * An ingest module template combines an ingest module factory with job * level ingest module settings to support the creation of any number of - * fully configured instances of a given ingest module. - * - * Note that an ingest module factory may be able to create multiple - * tpyes of ingest modules. + * fully configured instances of a given ingest module. An ingest module + * factory may be able to create multiple types of ingest modules. */ List enabledTemplates = settings.getEnabledIngestModuleTemplates(); /** - * Separate the ingest module templates into buckets based on the module + * Sort the ingest module templates into buckets based on the module * types the ingest module factory can create. A template may go into * more than one bucket. The buckets are actually maps of ingest module * factory class names to ingest module templates. The maps are used to * go from an ingest module factory class name read from the pipeline * configuration file to the corresponding ingest module template. * - * There are also two maps for each bucket. One map is for Java modules - * and the other one is for Jython modules. The templates are separated - * this way so that Java modules that are not in the pipeline config - * file can be placed before Jython modules. + * There are actually two maps for each module type bucket. One map is + * for Java modules and the other one is for Jython modules. The + * templates are separated this way so that Java modules that are not in + * the pipeline config file can be placed before the Jython modules. */ Map javaDataSourceModuleTemplates = new LinkedHashMap<>(); Map jythonDataSourceModuleTemplates = new LinkedHashMap<>(); @@ -347,23 +366,22 @@ final class IngestJobPipeline { Map jythonArtifactModuleTemplates = new LinkedHashMap<>(); for (IngestModuleTemplate template : enabledTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - addIngestModuleTermplateToMaps(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); + addIngestModuleTemplateToMaps(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); } if (template.isFileIngestModuleTemplate()) { - addIngestModuleTermplateToMaps(javaFileModuleTemplates, jythonFileModuleTemplates, template); + addIngestModuleTemplateToMaps(javaFileModuleTemplates, jythonFileModuleTemplates, template); } if (template.isDataArtifactIngestModuleTemplate()) { - addIngestModuleTermplateToMaps(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); + addIngestModuleTemplateToMaps(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); } } /** * Take the module templates that have pipeline configuration file * entries out of the buckets and put them in lists representing ingest - * task pipelines, in the order prescribed by the file. - * - * Note that the pipeline configuration file currently only supports - * specifying data source level and file ingest module pipeline layouts. + * task pipelines, in the order prescribed by the file. Note that the + * pipeline configuration file currently only supports specifying data + * source level and file ingest module pipeline layouts. */ IngestPipelinesConfiguration pipelineConfig = IngestPipelinesConfiguration.getInstance(); List firstStageDataSourceModuleTemplates = createPipelineFromConfigFile(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageOneDataSourceIngestPipelineConfig()); @@ -388,19 +406,11 @@ final class IngestJobPipeline { */ firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); - try { - int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); - for (int i = 0; i < numberOfFileIngestThreads; ++i) { - FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates); - fileIngestPipelinesQueue.put(pipeline); - fileIngestPipelines.add(pipeline); - } - } catch (InterruptedException ex) { - /* - * RC: This is not incorrect. If this thread is interrupted, the - * pipeline is incomplete and should not be used. We are currently relying on - */ - Thread.currentThread().interrupt(); + int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); + for (int i = 0; i < numberOfFileIngestThreads; ++i) { + FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates); + fileIngestPipelinesQueue.put(pipeline); + fileIngestPipelines.add(pipeline); } artifactIngestPipeline = new DataArtifactIngestPipeline(this, artifactModuleTemplates); } @@ -444,7 +454,16 @@ final class IngestJobPipeline { * @return The ID. */ long getId() { - return this.pipelineId; + return pipelineId; + } + + /** + * Gets the parent ingest job of this ingest pipeline. + * + * @return The ingest job. + */ + IngestJob getIngestJob() { + return job; } /** @@ -453,11 +472,11 @@ final class IngestJobPipeline { * @return The context name. */ String getExecutionContext() { - return this.settings.getExecutionContext(); + return settings.getExecutionContext(); } /** - * Gets the data source to be ingested by this ingest pipeline. + * Gets the data source to be analyzed by this ingest pipeline. * * @return The data source. */ @@ -465,6 +484,16 @@ final class IngestJobPipeline { return dataSource; } + /** + * Gets the subset of the files from the data source to be analyzed by this + * ingest pipeline. + * + * @return The files. + */ + List getFiles() { + return Collections.unmodifiableList(files); + } + /** * Queries whether or not unallocated space should be processed by this * ingest pipeline. @@ -472,7 +501,7 @@ final class IngestJobPipeline { * @return True or false. */ boolean shouldProcessUnallocatedSpace() { - return this.settings.getProcessUnallocatedSpace(); + return settings.getProcessUnallocatedSpace(); } /** @@ -481,7 +510,7 @@ final class IngestJobPipeline { * @return The filter. */ FilesSet getFileIngestFilter() { - return this.settings.getFileFilter(); + return settings.getFileFilter(); } /** @@ -491,20 +520,10 @@ final class IngestJobPipeline { * @return True or false. */ boolean hasIngestModules() { - return hasArtifactIngestModules() - || hasFileIngestModules() + return hasFileIngestModules() || hasFirstStageDataSourceIngestModules() - || hasSecondStageDataSourceIngestModules(); - } - - /** - * Checks to see if this ingest pipeline has at least one data artifact - * ingest module. - * - * @return True or false. - */ - private boolean hasArtifactIngestModules() { - return (artifactIngestPipeline.isEmpty() == false); + || hasSecondStageDataSourceIngestModules() + || hasArtifactIngestModules(); } /** @@ -513,17 +532,17 @@ final class IngestJobPipeline { * * @return True or false. */ - private boolean hasFirstStageDataSourceIngestModules() { + boolean hasFirstStageDataSourceIngestModules() { return (firstStageDataSourceIngestPipeline.isEmpty() == false); } /** * Checks to see if this ingest pipeline has at least one second stage data - * source level ingest modules. + * source level ingest module. * * @return True or false. */ - private boolean hasSecondStageDataSourceIngestModules() { + boolean hasSecondStageDataSourceIngestModules() { return (secondStageDataSourceIngestPipeline.isEmpty() == false); } @@ -533,7 +552,7 @@ final class IngestJobPipeline { * * @return True or false. */ - private boolean hasFileIngestModules() { + boolean hasFileIngestModules() { if (!fileIngestPipelines.isEmpty()) { /* * Note that the file ingest task pipelines are identical. @@ -543,24 +562,30 @@ final class IngestJobPipeline { return false; } + /** + * Checks to see if this ingest pipeline has at least one artifact ingest + * module. + * + * @return True or false. + */ + boolean hasArtifactIngestModules() { + return (artifactIngestPipeline.isEmpty() == false); + } + /** * Starts up this ingest pipeline. * * @return A collection of ingest module startup errors, empty on success. */ List startUp() { - if (dataSource == null) { - throw new IllegalStateException("Ingest started before setting data source"); //NON-NLS - } - List errors = startUpIngestTaskPipelines(); if (errors.isEmpty()) { recordIngestJobStartUpInfo(); - if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules()) { - if (job.getIngestMode() == IngestJob.Mode.BATCH) { - startFirstStageInBatchMode(); - } else { + if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasArtifactIngestModules()) { + if (job.getIngestMode() == IngestJob.Mode.STREAMING) { startFirstStageInStreamingMode(); + } else { + startFirstStage(); } } else if (hasSecondStageDataSourceIngestModules()) { startSecondStage(); @@ -595,7 +620,7 @@ final class IngestJobPipeline { * * @param moduleTemplate The ingest module template. * - * @return The ingest module type. may be IngestModuleType.MULTIPLE. + * @return The ingest module type, may be IngestModuleType.MULTIPLE. */ private IngestModuleType getIngestModuleTemplateType(IngestModuleTemplate moduleTemplate) { IngestModuleType type = null; @@ -653,7 +678,7 @@ final class IngestJobPipeline { /** * Starts up an ingest task pipeline. If there are any start up errors, the - * pipeline is imediately shut down. + * pipeline is immediately shut down. * * @param pipeline The ingest task pipeline to start up. * @@ -671,20 +696,21 @@ final class IngestJobPipeline { } /** - * Schedules the first stage results, data source level, and file ingest - * tasks for the data source. + * Starts the first stage of this pipeline in batch mode. In batch mode, all + * of the files in the data source (excepting carved and derived files) have + * already been added to the case database by the data source processor. */ - private void startFirstStageInBatchMode() { - logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS - stage = Stages.FIRST_STAGE_ALL_TASKS; - + private void startFirstStage() { if (hasFileIngestModules()) { /* - * The estimated number of files remaining to be processed is used - * for ingest snapshots and for the file ingest progress bar. + * Do a count of the files the data source processor has added to + * the case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a + * GUI. */ + long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; synchronized (fileIngestProgressLock) { - estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); + estimatedFilesToProcess = filesToProcess; } } @@ -704,7 +730,7 @@ final class IngestJobPipeline { } } - /** + /* * Make the first stage data source level ingest pipeline the current * data source level pipeline. */ @@ -712,55 +738,39 @@ final class IngestJobPipeline { currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; } - /** - * Schedule the first stage ingest tasks. - */ - if (hasArtifactIngestModules()) { - /* - * Create ingest tasks for data artifacts already in the case - * database. Additional tasks will be created as other ingest - * modules add data aritfacts. - */ - taskScheduler.scheduleDataArtifactIngestTasks(this); - } - if (hasFirstStageDataSourceIngestModules() && hasFileIngestModules()) { - taskScheduler.scheduleDataSourceAndFileIngestTasks(this); - } else if (hasFirstStageDataSourceIngestModules()) { - taskScheduler.scheduleDataSourceIngestTask(this); - } else if (hasFileIngestModules() && !files.isEmpty()) { - taskScheduler.scheduleFileIngestTasks(this, files); + synchronized (stageTransitionLock) { + logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS + stage = Stages.FIRST_STAGE; + /** - * No data source ingest task has been scheduled for this stage, it - * is possible that no artifact tasks were scheduled, and it is also - * possible that no file ingest tasks were scheduled when the task - * scheduler applied the file ingest filter. In this special case in - * which there are no ingest tasks to do, an ingest thread will - * never get to check for completion of this stage of the job, so do - * it now so there is not a do-nothing ingest job that lives - * forever. + * Schedule the first stage ingest tasks and then immediately check + * for stage completion. This is necessary because it is possible + * that zero tasks will actually make it to task execution due to + * the file filter or other ingest job settings. In that case, there + * will never be a stage completion check in an ingest thread + * executing an ingest task, so such a job would run forever without + * the check here. */ + taskScheduler.scheduleIngestTasks(this); checkForStageCompleted(); } } /** - * Schedules the first stage results ingest tasks and prepares for streaming - * file ingest (used for streaming ingest only). Does not schedule any file - * tasks - those will come from calls to addStreamingIngestFiles(). + * Starts the first stage of this pipeline in streaming mode. In streaming + * mode, the data source processor streams files into the pipeline as it + * adds them to the case database and only adds the data source to the + * pipeline after all of the files have been streamed in. See + * addStreamingIngestFiles() and addStreamingIngestDataSource(). */ private void startFirstStageInStreamingMode() { - logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS - stage = Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY; - if (hasFileIngestModules()) { - /* - * The estimated number of files remaining to be processed is used - * for ingest snapshots and for the file ingest progress bar. - * However, for streaming ingest, it cannot be calculated until the - * data source is added to this pipeline. Start with zero to signal - * an unknown value. - */ synchronized (fileIngestProgressLock) { + /* + * Start with zero to signal an unknown value. This estimate + * will be used for ingest progress snapshots and for the file + * ingest progress bar if running with a GUI. + */ estimatedFilesToProcess = 0; } } @@ -770,9 +780,6 @@ final class IngestJobPipeline { * hand corner of the main application window. */ if (doUI) { - if (hasArtifactIngestModules()) { - startArtifactIngestProgressBar(); - } if (hasFileIngestModules()) { /* * Note that because estimated files remaining to process has @@ -781,51 +788,49 @@ final class IngestJobPipeline { */ startFileIngestProgressBar(); } + if (hasArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } } - /** - * Schedule the first stage ingest tasks for streaming mode. For - * streaming ingest, file ingest tasks are created as the files are - * streamed into this pipeline, so there are no tasks to schedule yet. - * Also, for streaming ingest, the data source is not added to the - * pipeline right away, so there is no data source level task to - * schedule yet. So only result ingest tasks can be scheduled at this - * time. - */ - if (hasArtifactIngestModules()) { - /* - * Create ingest tasks for data artifacts already in the case - * database. Additional tasks will be created as other ingest - * modules add data aritfacts. - */ - taskScheduler.scheduleDataArtifactIngestTasks(this); + synchronized (stageTransitionLock) { + logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS + stage = Stages.FIRST_STAGE_STREAMING; + if (hasArtifactIngestModules()) { + /* + * Schedule artifact ingest tasks for any artifacts currently in + * the case database. This needs to be done before any files or + * the data source are streamed in to avoid analyzing data + * artifacts added to the case database by the data source level + * or file level ingest tasks. + */ + taskScheduler.scheduleDataArtifactIngestTasks(this); + } } } /** - * Starts a data source level ingest. Used for streaming ingest, in which - * the data source is not ready when ingest starts. + * Start data source ingest. Used for streaming ingest when the data source + * is not ready when ingest starts. */ - private void startDataSourceTaskInStreamingMode() { + void addStreamingIngestDataSource() { /* - * Now that the data source processor analysis of the data source is - * complete, an estimate of the files remaining to be processed can be - * calculated and the file ingest progress bar in the lower right hand - * corner of the main application window Fcan be switched from - * indeterminate to determinate. + * Do a count of the files the data source processor has added to the + * case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a GUI. + * The count will be off by any streamed files that have already been + * analyzed. */ + long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; synchronized (fileIngestProgressLock) { - estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); - if (doUI && fileIngestProgress != null) { - fileIngestProgress.switchToDeterminate((int) estimatedFilesToProcess); - } + estimatedFilesToProcess = filesToProcess; } + /* + * If running with a GUI, start ingest progress bars in the lower right + * hand corner of the main application window. + */ if (doUI) { - /** - * Start the first stage data source ingest progress bar in the - * lower right hand corner of the main application window. - */ if (hasFirstStageDataSourceIngestModules()) { startDataSourceIngestProgressBar(); } @@ -839,10 +844,22 @@ final class IngestJobPipeline { this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; } - logInfoMessage("Scheduling first stage data source level ingest task in streaming mode"); //NON-NLS - synchronized (this.stageTransitionLock) { - stage = IngestJobPipeline.Stages.FIRST_STAGE_ALL_TASKS; - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); + synchronized (stageTransitionLock) { + logInfoMessage("Adding the data source in streaming mode"); //NON-NLS + stage = IngestJobPipeline.Stages.FIRST_STAGE; + if (hasFirstStageDataSourceIngestModules()) { + IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + /* + * If no data source level ingest task is scheduled at this time + * and all of the file level and artifact ingest tasks scheduled + * when streaming began have already executed, there will never + * be a stage completion check in an ingest thread executing an + * ingest task, so such a job would run forever without the + * check here. + */ + checkForStageCompleted(); + } } } @@ -850,15 +867,17 @@ final class IngestJobPipeline { * Starts the second stage ingest task pipelines. */ private void startSecondStage() { - logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), job.getId())); //NON-NLS - stage = IngestJobPipeline.Stages.SECOND_STAGE; if (doUI) { startDataSourceIngestProgressBar(); } synchronized (dataSourceIngestPipelineLock) { currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; } - taskScheduler.scheduleDataSourceIngestTask(this); + synchronized (stageTransitionLock) { + logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), job.getId())); //NON-NLS + stage = IngestJobPipeline.Stages.SECOND_STAGE; + taskScheduler.scheduleDataSourceIngestTask(this); + } } /** @@ -866,17 +885,17 @@ final class IngestJobPipeline { */ private void startArtifactIngestProgressBar() { if (doUI) { - synchronized (resultsIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.resultsIngest.displayName", this.dataSource.getName()); - resultsIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() { + synchronized (artifactIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", this.dataSource.getName()); + artifactIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { IngestJobPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); return true; } }); - resultsIngestProgress.start(); - resultsIngestProgress.switchToIndeterminate(); + artifactIngestProgressBar.start(); + artifactIngestProgressBar.switchToIndeterminate(); } } } @@ -890,7 +909,7 @@ final class IngestJobPipeline { String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", this.dataSource.getName()); - this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() { + this.dataSourceIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { // If this method is called, the user has already pressed @@ -910,8 +929,8 @@ final class IngestJobPipeline { return true; } }); - this.dataSourceIngestProgress.start(); - this.dataSourceIngestProgress.switchToIndeterminate(); + this.dataSourceIngestProgressBar.start(); + this.dataSourceIngestProgressBar.switchToIndeterminate(); } } } @@ -925,7 +944,7 @@ final class IngestJobPipeline { String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()); - this.fileIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() { + this.fileIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { // If this method is called, the user has already pressed @@ -936,8 +955,8 @@ final class IngestJobPipeline { return true; } }); - this.fileIngestProgress.start(); - this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); + this.fileIngestProgressBar.start(); + this.fileIngestProgressBar.switchToDeterminate((int) this.estimatedFilesToProcess); } } } @@ -948,51 +967,15 @@ final class IngestJobPipeline { */ private void checkForStageCompleted() { synchronized (stageTransitionLock) { - /* - * Note that there is nothing to do here for the - * FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY stage for streaming - * ingest, because we need to wait for the data source to be added - * to the job and the transition to FIRST_STAGE_ALL_TASKS. - */ - if (stage != Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY - && IngestJobPipeline.taskScheduler.currentTasksAreCompleted(this)) { + if (stage == Stages.FIRST_STAGE_STREAMING) { + return; + } + if (taskScheduler.currentTasksAreCompleted(this)) { switch (stage) { - case FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY: - /* - * Nothing to do here, need to wait for the data source - * to be added to the job and the transition to - * FIRST_STAGE_ALL_TASKS. - */ - break; - case FIRST_STAGE_ALL_TASKS: - /* - * Tasks completed: - * - * 1. The first stage data source level ingest task (if - * any). - * - * 2. All file ingest tasks, including tasks for - * carved/derived files submitted by ingest modules in - * the first stage via IngestJobContext,addFilesToJob(). - * - * 3. The results tasks for results in the case database - * when the job started, plus any result tasks submitted - * by the ingest modules in other tasks in the first - * stage via IngestJobContext.addDataArtifactsToJob(). - */ + case FIRST_STAGE: finishFirstStage(); break; case SECOND_STAGE: - /* - * Tasks completed: - * - * 1. The second stage data source level ingest task (if - * any). - * - * 2. The results tasks for any results added by second - * stage ingest modules via - * IngestJobContext.addDataArtifactsToJob(). - */ shutDown(); break; } @@ -1013,8 +996,8 @@ final class IngestJobPipeline { shutDownIngestTaskPipeline(pipeline); } - finishProgressBar(dataSourceIngestProgress, dataSourceIngestProgressLock); - finishProgressBar(fileIngestProgress, fileIngestProgressLock); + finishProgressBar(dataSourceIngestProgressBar, dataSourceIngestProgressLock); + finishProgressBar(fileIngestProgressBar, fileIngestProgressLock); if (!cancelled && hasSecondStageDataSourceIngestModules()) { startSecondStage(); @@ -1027,38 +1010,40 @@ final class IngestJobPipeline { * Shuts down the ingest pipelines and progress bars for this job. */ private void shutDown() { - logInfoMessage("Finished all tasks"); //NON-NLS - stage = IngestJobPipeline.Stages.FINALIZATION; + synchronized (stageTransitionLock) { + logInfoMessage("Finished all tasks"); //NON-NLS + stage = IngestJobPipeline.Stages.FINALIZATION; - shutDownIngestTaskPipeline(currentDataSourceIngestPipeline); - shutDownIngestTaskPipeline(artifactIngestPipeline); + shutDownIngestTaskPipeline(currentDataSourceIngestPipeline); + shutDownIngestTaskPipeline(artifactIngestPipeline); - finishProgressBar(dataSourceIngestProgress, dataSourceIngestProgressLock); - finishProgressBar(fileIngestProgress, fileIngestProgressLock); - finishProgressBar(resultsIngestProgress, resultsIngestProgressLock); + finishProgressBar(dataSourceIngestProgressBar, dataSourceIngestProgressLock); + finishProgressBar(fileIngestProgressBar, fileIngestProgressLock); + finishProgressBar(artifactIngestProgressBar, artifactIngestProgressLock); - if (ingestJobInfo != null) { - if (cancelled) { - try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + if (ingestJobInfo != null) { + if (cancelled) { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } + } else { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } } - } else { try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); + ingestJobInfo.setEndDateTime(new Date()); } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } } - try { - ingestJobInfo.setEndDateTime(new Date()); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); - } + + job.notifyIngestPipelineShutDown(this); } - - job.notifyIngestPipelineShutDown(this); } /** @@ -1139,9 +1124,9 @@ final class IngestJobPipeline { * Data source-level processing is finished for this stage. */ synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgress != null) { - dataSourceIngestProgress.finish(); - dataSourceIngestProgress = null; + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; } } } @@ -1187,9 +1172,9 @@ final class IngestJobPipeline { * right hand corner of the main application window. */ if (processedFiles <= estimatedFilesToProcess) { - fileIngestProgress.progress(file.getName(), (int) processedFiles); + fileIngestProgressBar.progress(file.getName(), (int) processedFiles); } else { - fileIngestProgress.progress(file.getName(), (int) estimatedFilesToProcess); + fileIngestProgressBar.progress(file.getName(), (int) estimatedFilesToProcess); } filesInProgress.add(file.getName()); } @@ -1212,9 +1197,9 @@ final class IngestJobPipeline { */ filesInProgress.remove(file.getName()); if (filesInProgress.size() > 0) { - fileIngestProgress.progress(filesInProgress.get(0)); + fileIngestProgressBar.progress(filesInProgress.get(0)); } else { - fileIngestProgress.progress(""); + fileIngestProgressBar.progress(""); } } } @@ -1252,32 +1237,23 @@ final class IngestJobPipeline { } /** - * Adds some subset of the "streamed" files for a stremaing ingest job to + * Adds some subset of the "streamed" files for a streaming ingest job to * this pipeline after startUp() has been called. * * @param fileObjIds The object IDs of the files. */ void addStreamingIngestFiles(List fileObjIds) { - if (hasFileIngestModules()) { - if (stage.equals(Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY)) { - IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + synchronized (stageTransitionLock) { + if (hasFileIngestModules()) { + if (stage.equals(Stages.FIRST_STAGE_STREAMING)) { + IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + } } } } - /** - * Adds the data source for a streaming ingest job to this pipeline after - * startUp() has been called. Intended to be called after the data source - * processor has finished its processing (i.e., all the file system files - * for the data source are in the case database). - */ - void addStreamingIngestDataSource() { - startDataSourceTaskInStreamingMode(); - checkForStageCompleted(); - } - /** * Adds additional files (e.g., extracted or carved files) for any type of * ingest job to this pipeline after startUp() has been called. Not @@ -1286,21 +1262,23 @@ final class IngestJobPipeline { * @param files A list of the files to add. */ void addFiles(List files) { - if (stage.equals(Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY) - || stage.equals(Stages.FIRST_STAGE_ALL_TASKS)) { - taskScheduler.fastTrackFileIngestTasks(this, files); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); - } + synchronized (stageTransitionLock) { + if (stage.equals(Stages.FIRST_STAGE_STREAMING) + || stage.equals(Stages.FIRST_STAGE)) { + taskScheduler.fastTrackFileIngestTasks(this, files); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + } - /** - * The intended clients of this method are ingest modules running code - * in an ingest thread that is holding a reference to a "primary" ingest - * task that was the source of the files, in which case a completion - * check would not be necessary, so this is a bit of defensive - * programming. - */ - checkForStageCompleted(); + /** + * The intended clients of this method are ingest modules running + * code in an ingest thread that is holding a reference to a + * "primary" ingest task that was the source of the files, in which + * case a completion check would not be necessary, so this is a bit + * of defensive programming. + */ + checkForStageCompleted(); + } } /** @@ -1310,22 +1288,24 @@ final class IngestJobPipeline { * @param artifacts */ void addDataArtifacts(List artifacts) { - if (stage.equals(Stages.FIRST_STAGE_FILE_AND_RESULTS_TASKS_ONLY) - || stage.equals(Stages.FIRST_STAGE_ALL_TASKS) - || stage.equals(Stages.SECOND_STAGE)) { - taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); - } + synchronized (stageTransitionLock) { + if (stage.equals(Stages.FIRST_STAGE_STREAMING) + || stage.equals(Stages.FIRST_STAGE) + || stage.equals(Stages.SECOND_STAGE)) { + taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + } - /** - * The intended clients of this method are ingest modules running code - * in an ingest thread that is holding a reference to a "primary" ingest - * task that was the source of the files, in which case a completion - * check would not be necessary, so this is a bit of defensive - * programming. - */ - checkForStageCompleted(); + /** + * The intended clients of this method are ingest modules running + * code in an ingest thread that is holding a reference to a + * "primary" ingest task that was the source of the files, in which + * case a completion check would not be necessary, so this is a bit + * of defensive programming. + */ + checkForStageCompleted(); + } } /** @@ -1337,7 +1317,7 @@ final class IngestJobPipeline { void updateDataSourceIngestProgressBarDisplayName(String displayName) { if (this.doUI && !this.cancelled) { synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.setDisplayName(displayName); + this.dataSourceIngestProgressBar.setDisplayName(displayName); } } } @@ -1353,8 +1333,8 @@ final class IngestJobPipeline { void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { if (this.doUI && !this.cancelled) { synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToDeterminate(workUnits); + if (null != this.dataSourceIngestProgressBar) { + this.dataSourceIngestProgressBar.switchToDeterminate(workUnits); } } } @@ -1368,8 +1348,8 @@ final class IngestJobPipeline { void switchDataSourceIngestProgressBarToIndeterminate() { if (this.doUI && !this.cancelled) { synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToIndeterminate(); + if (null != this.dataSourceIngestProgressBar) { + this.dataSourceIngestProgressBar.switchToIndeterminate(); } } } @@ -1384,8 +1364,8 @@ final class IngestJobPipeline { void advanceDataSourceIngestProgressBar(int workUnits) { if (doUI && !cancelled) { synchronized (dataSourceIngestProgressLock) { - if (null != dataSourceIngestProgress) { - dataSourceIngestProgress.progress("", workUnits); + if (null != dataSourceIngestProgressBar) { + dataSourceIngestProgressBar.progress("", workUnits); } } } @@ -1400,8 +1380,8 @@ final class IngestJobPipeline { void advanceDataSourceIngestProgressBar(String currentTask) { if (doUI && !cancelled) { synchronized (dataSourceIngestProgressLock) { - if (null != dataSourceIngestProgress) { - dataSourceIngestProgress.progress(currentTask); + if (null != dataSourceIngestProgressBar) { + dataSourceIngestProgressBar.progress(currentTask); } } } @@ -1418,7 +1398,7 @@ final class IngestJobPipeline { void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { if (this.doUI && !this.cancelled) { synchronized (this.fileIngestProgressLock) { - this.dataSourceIngestProgress.progress(currentTask, workUnits); + this.dataSourceIngestProgressBar.progress(currentTask, workUnits); } } } @@ -1453,8 +1433,8 @@ final class IngestJobPipeline { * is pressed. */ synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; + this.dataSourceIngestProgressBar.finish(); + this.dataSourceIngestProgressBar = null; this.startDataSourceIngestProgressBar(); } } @@ -1494,16 +1474,16 @@ final class IngestJobPipeline { if (this.doUI) { synchronized (this.dataSourceIngestProgressLock) { - if (null != dataSourceIngestProgress) { - dataSourceIngestProgress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", this.dataSource.getName())); - dataSourceIngestProgress.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); + if (null != dataSourceIngestProgressBar) { + dataSourceIngestProgressBar.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", this.dataSource.getName())); + dataSourceIngestProgressBar.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); } } synchronized (this.fileIngestProgressLock) { - if (null != this.fileIngestProgress) { - this.fileIngestProgress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName())); - this.fileIngestProgress.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); + if (null != this.fileIngestProgressBar) { + this.fileIngestProgressBar.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName())); + this.fileIngestProgressBar.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); } } } @@ -1620,7 +1600,7 @@ final class IngestJobPipeline { estimatedFilesToProcessCount = this.estimatedFilesToProcess; snapShotTime = new Date().getTime(); } - tasksSnapshot = IngestJobPipeline.taskScheduler.getTasksSnapshotForJob(pipelineId); + tasksSnapshot = taskScheduler.getTasksSnapshotForJob(pipelineId); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index e814687218..f14f9518c1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -186,10 +186,10 @@ public class IngestManager implements IngestProgressSnapshotProvider { fileLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } - + resultIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-results-ingest-%d").build()); //NON-NLS; threadId = nextIngestManagerTaskId.incrementAndGet(); - dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getResultIngestTaskQueue())); + dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getResultIngestTaskQueue())); // RJCTODO // ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); // RJCTODO: Where is the shut down code? @@ -464,7 +464,11 @@ public class IngestManager implements IngestProgressSnapshotProvider { ingestJobsById.put(job.getId(), job); } IngestManager.logger.log(Level.INFO, "Starting ingest job {0}", job.getId()); //NON-NLS - errors = job.start(); + try { + errors = job.start(); + } catch (InterruptedException ex) { + return new IngestJobStartResult(null, new IngestManagerException("Interrupted while starting ingest", ex), errors); //NON-NLS + } if (errors.isEmpty()) { this.fireIngestJobStarted(job.getId()); } else { @@ -501,7 +505,8 @@ public class IngestManager implements IngestProgressSnapshotProvider { * * @param job The completed job. */ - void finishIngestJob(IngestJob job) { + void finishIngestJob(IngestJob job + ) { long jobId = job.getId(); synchronized (ingestJobsById) { ingestJobsById.remove(jobId); @@ -793,31 +798,41 @@ public class IngestManager implements IngestProgressSnapshotProvider { * @param currentModuleName The display name of the currently processing * module. */ - void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) { - ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestPipeline().getId(), ingestModuleDisplayName, task.getDataSource())); + void setIngestTaskProgress(DataSourceIngestTask task, String currentModuleName) { + IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); + IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJobPipeline().getId(), currentModuleName, task.getDataSource()); + ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); + + /* + * Update the total run time for the PREVIOUS ingest module in the + * pipeline, which has now finished its processing for the task. + */ + incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); } /** * Updates the ingest progress snapshot when a new ingest module starts * working on a file ingest task. * - * @param task The file level ingest job task that was - * started - * @param ingestModuleDisplayName The dislpay name of the file level ingest - * module that has started processing the - * task. + * @param task The file ingest task. + * @param currentModuleName The display name of the currently processing + * module. */ - void setIngestTaskProgress(FileIngestTask task, String ingestModuleDisplayName) { + void setIngestTaskProgress(FileIngestTask task, String currentModuleName) { IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); IngestThreadActivitySnapshot newSnap; try { - newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestPipeline().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile()); + newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJobPipeline().getId(), currentModuleName, task.getDataSource(), task.getFile()); } catch (TskCoreException ex) { - // In practice, this task would never have been enqueued or processed since the file - // lookup would have failed. - newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestPipeline().getId(), ingestModuleDisplayName, task.getDataSource()); + logger.log(Level.SEVERE, "Error getting file from file ingest task", ex); + newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJobPipeline().getId(), currentModuleName, task.getDataSource()); } ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); + + /* + * Update the total run time for the PREVIOUS ingest module in the + * pipeline, which has now finished its processing for the task. + */ incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java index 1ce3770360..84a5cb6bfd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index 7dbed1ffbf..5c4b4148b4 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -51,7 +51,7 @@ abstract class IngestTask { * * @return The ingest job pipeline. */ - IngestJobPipeline getIngestPipeline() { + IngestJobPipeline getIngestJobPipeline() { return ingestJobPipeline; } @@ -61,7 +61,7 @@ abstract class IngestTask { * @return The data source. */ Content getDataSource() { - return getIngestPipeline().getDataSource(); + return getIngestJobPipeline().getDataSource(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java index 081faa1976..abb0e4cf54 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java @@ -22,44 +22,53 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Optional; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; /** - * A generic abstract superclass for pipelines of ingest modules for executing - * ingest tasks for an ingest job. The type parameter of the generic is an - * ingest task type. Some examples of ingest task types: data source level - * ingest tasks, file ingest tasks, data artifact ingest tasks, etc. Subclasses - * need to implement a specialization of the inner PipelineModule generic - * abstract superclass for the type of ingest modules that make up the pipeline. + * An abstract superclass for pipelines of ingest modules that execute ingest + * tasks for an ingest job. + * + * Conceptually, an ingest job pipeline is divided into one or more "sub + * pipelines" that are actually ingest task pipelines of varying types. Thus, + * the type parameter of this generic is an ingest task type. + * + * IMPORTANT: Subclasses need to both extend this class, and to implement a + * specialization of the inner PipelineModule abstract superclass. * * @param The ingest task type. */ +@ThreadSafe abstract class IngestTaskPipeline { - private final IngestJobPipeline ingestPipeline; + private final IngestJobPipeline ingestJobPipeline; + @GuardedBy("this") private final List moduleTemplates; + @GuardedBy("this") private final List> modules; private volatile Date startTime; private volatile boolean running; private volatile PipelineModule currentModule; /** - * Constructs a generic abstract superclass for pipelines of ingest modules - * for executing ingest tasks for an ingest job. The type parameter of the - * generic is an ingest task type. Some examples of ingest task types: data - * source level ingest tasks, file ingest tasks, data artifact ingest tasks, - * etc. Subclasses need to implement a specialization of the inner - * PipelineModule generic abstract superclass for the type of ingest modules - * that make up the pipeline. + * Constructs the superclass part of a pipeline of ingest modules that + * executes ingest tasks for an ingest job. * - * @param ingestPipeline The parent ingest pipeline for this ingest task - * pipeline. + * @param ingestPipeline The parent ingest job pipeline for this ingest + * task pipeline. * @param moduleTemplates The ingest module templates that define this - * ingest task pipeline. + * ingest task pipeline. May be an empty list. */ IngestTaskPipeline(IngestJobPipeline ingestPipeline, List moduleTemplates) { - this.ingestPipeline = ingestPipeline; + this.ingestJobPipeline = ingestPipeline; + /* + * The creation of ingest modules from the ingest module templates has + * been deliberately deferred to the startUp() method so that any and + * all errors in module construction or start up can be reported to the + * client code. + */ this.moduleTemplates = moduleTemplates; modules = new ArrayList<>(); } @@ -70,13 +79,13 @@ abstract class IngestTaskPipeline { * * @return True or false. */ - boolean isEmpty() { + synchronized boolean isEmpty() { return modules.isEmpty(); } /** - * Queries whether or not this ingest task pipeline is running, i.e., - * started up and not yet shut down. + * Queries whether or not this ingest task pipeline is running, i.e., the + * startUp() method has been called and the shutDown() has not been called. * * @return True or false. */ @@ -85,42 +94,55 @@ abstract class IngestTaskPipeline { } /** - * Starts up this ingest task pipeline by starting the ingest modules in the - * pipeline. + * Starts up this ingest task pipeline by calling the startUp() methods of + * the ingest modules in the pipeline. * * @return A list of ingest module start up errors, possibly empty. */ - List startUp() { - createIngestModules(moduleTemplates); - return startUpIngestModules(); + synchronized List startUp() { + List errors = new ArrayList<>(); + if (!running) { + /* + * The creation of ingest modules from the ingest module templates + * has been deliberately deferred to the startUp() method so that + * any and all errors in module construction or start up can be + * reported to the client code. + */ + createIngestModules(moduleTemplates); + errors.addAll(startUpIngestModules()); + } else { + errors.add(new IngestModuleError("Ingest Task Pipeline", new IngestTaskPipelineException("Pipeline already started"))); //NON-NLS + } + return errors; } /** - * Creates the ingest modules for this ingest task pipeline from ingest - * module templates. + * Creates the ingest modules for this ingest task pipeline from the given + * ingest module templates. * - * @param moduleTemplates The ingest module templates avaialble to this - * pipeline. + * @param moduleTemplates The ingest module templates. */ private void createIngestModules(List moduleTemplates) { - for (IngestModuleTemplate template : moduleTemplates) { - Optional> module = acceptModuleTemplate(template); - if (module.isPresent()) { - modules.add(module.get()); + if (modules.isEmpty()) { + for (IngestModuleTemplate template : moduleTemplates) { + Optional> module = acceptModuleTemplate(template); + if (module.isPresent()) { + modules.add(module.get()); + } } } } /** - * Determines if the type of ingest module that can be created from a given - * ingest module template should be added to this ingest task pipeline. If - * so, the ingest module is created and returned. + * Determines if one of the types of ingest modules that can be created from + * a given ingest module template should be added to this ingest task + * pipeline. If so, the ingest module is created and returned. * * @param template The ingest module template to be used or ignored, as * appropriate to the pipeline type. * - * @return An Optional that is either empty or contains a newly created and - * wrapped ingest module. + * @return An Optional that is either empty or contains a newly created + * ingest module of type T, wrapped in a PipelineModule decorator. */ abstract Optional> acceptModuleTemplate(IngestModuleTemplate template); @@ -130,13 +152,19 @@ abstract class IngestTaskPipeline { * @return A list of ingest module start up errors, possibly empty. */ private List startUpIngestModules() { + List errors = new ArrayList<>(); startTime = new Date(); running = true; - List errors = new ArrayList<>(); for (PipelineModule module : modules) { try { - module.startUp(new IngestJobContext(ingestPipeline)); - } catch (Throwable ex) { // Catch-all exception firewall + module.startUp(new IngestJobContext(ingestJobPipeline)); + } catch (Throwable ex) { + /* + * A catch-all exception firewall. Start up errors for all of + * the ingest modules, whether checked exceptions or runtime + * exceptions, are reported to allow correction of all of the + * error conditions in one go. + */ errors.add(new IngestModuleError(module.getDisplayName(), ex)); } } @@ -158,54 +186,65 @@ abstract class IngestTaskPipeline { } /** - * Does any preparation required before performing a task. - * - * @param task The task. - * - * @throws IngestTaskPipelineException Thrown if there is an error preparing - * to perform the task. - */ - abstract void prepareForTask(T task) throws IngestTaskPipelineException; - - /** - * Executes an ingest task using the ingest modules in this ingest task - * pipeline. + * Executes an ingest task by calling the process() methods of the ingest + * modules in this ingest task pipeline. * * @param task The task. * * @return A list of ingest module task processing errors, possibly empty. */ - List executeTask(T task) { + synchronized List executeTask(T task) { List errors = new ArrayList<>(); - if (!this.ingestPipeline.isCancelled()) { + if (running) { + if (!ingestJobPipeline.isCancelled()) { + try { + prepareForTask(task); + } catch (IngestTaskPipelineException ex) { + errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS + return errors; + } + for (PipelineModule module : modules) { + try { + currentModule = module; + currentModule.setProcessingStartTime(); + module.executeTask(ingestJobPipeline, task); + } catch (Throwable ex) { + /* + * A catch-all exception firewall. Note that a runtime + * exception from a single module does not stop + * processing of the task by the other modules in the + * pipeline. + */ + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + if (ingestJobPipeline.isCancelled()) { + break; + } + } + } try { - prepareForTask(task); + cleanUpAfterTask(task); } catch (IngestTaskPipelineException ex) { errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS - return errors; } - for (PipelineModule module : modules) { - try { - currentModule = module; - currentModule.setProcessingStartTime(); - module.executeTask(ingestPipeline, task); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - if (ingestPipeline.isCancelled()) { - break; - } - } - } - try { - cleanUpAfterTask(task); - } catch (IngestTaskPipelineException ex) { - errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS + } else { + errors.add(new IngestModuleError("Ingest Task Pipeline", new IngestTaskPipelineException("Pipeline not started or shut down"))); //NON-NLS } currentModule = null; return errors; } + /** + * Does any task type specific preparation required before executing an + * ingest task. + * + * @param task The task. + * + * @throws IngestTaskPipelineException Thrown if there is an error preparing + * to execute the task. + */ + abstract void prepareForTask(T task) throws IngestTaskPipelineException; + /** * Gets the currently running ingest module. * @@ -216,21 +255,11 @@ abstract class IngestTaskPipeline { } /** - * Does any clean up required after performing a task. - * - * @param task The task. - * - * @throws IngestTaskPipelineException Thrown if there is an error cleaning - * up after performing the task. - */ - abstract void cleanUpAfterTask(T task) throws IngestTaskPipelineException; - - /** - * Shuts down all of the ingest modules in this ingest task pipeline. + * Shuts down all of the ingest modules in this pipeline. * * @return A list of shut down errors, possibly empty. */ - List shutDown() { + synchronized List shutDown() { List errors = new ArrayList<>(); if (running == true) { for (PipelineModule module : modules) { @@ -256,8 +285,22 @@ abstract class IngestTaskPipeline { } /** - * An abstract superclass for a wrapper that adds ingest infrastructure + * Does any task type specific clean up required after executing an ingest + * task. + * + * @param task The task. + * + * @throws IngestTaskPipelineException Thrown if there is an error cleaning + * up after performing the task. + */ + abstract void cleanUpAfterTask(T task) throws IngestTaskPipelineException; + + /** + * An abstract superclass for a decorator that adds ingest infrastructure * operations to an ingest module. + * + * IMPORTANT: Subclasses of IngestTaskPipeline need to implement a + * specialization this class */ static abstract class PipelineModule implements IngestModule { @@ -266,7 +309,7 @@ abstract class IngestTaskPipeline { private volatile Date processingStartTime; /** - * Constructs an instance of an abstract superclass for a wrapper that + * Constructs an instance of an abstract superclass for a decorator that * adds ingest infrastructure operations to an ingest module. * * @param module The ingest module to be wrapped. @@ -279,7 +322,7 @@ abstract class IngestTaskPipeline { } /** - * Gets the class name of the wrapped ingest module. + * Gets the class name of the decorated ingest module. * * @return The class name. */ @@ -288,7 +331,7 @@ abstract class IngestTaskPipeline { } /** - * Gets the display name of the wrapped ingest module. + * Gets the display name of the decorated ingest module. * * @return The display name. */ @@ -297,7 +340,7 @@ abstract class IngestTaskPipeline { } /** - * Sets the processing start time for the wrapped module to the system + * Sets the processing start time for the decorated module to the system * time when this method is called. */ void setProcessingStartTime() { @@ -305,7 +348,7 @@ abstract class IngestTaskPipeline { } /** - * Gets the the processing start time for the wrapped module. + * Gets the the processing start time for the decorated module. * * @return The start time, will be null if the module has not started * processing the data source yet. @@ -320,12 +363,12 @@ abstract class IngestTaskPipeline { } /** - * Executes an ingest task using the process() method of the wrapped + * Executes an ingest task using the process() method of the decorated * module. * - * @param ingestJobPipeline The ingest pipeline that owns the ingest + * @param ingestJobPipeline The ingest job pipeline that owns the ingest * task pipeline this module belongs to. - * @param task The task to process. + * @param task The task to execute. * * @throws IngestModuleException Exception thrown if there is an error * performing the task. @@ -340,19 +383,31 @@ abstract class IngestTaskPipeline { } /** - * An exception for the use of ingest task pipelines. + * An exception thrown by an ingest task pipeline. */ public static class IngestTaskPipelineException extends Exception { private static final long serialVersionUID = 1L; + /** + * Constructs an exception to be thrown by an ingest task pipeline. + * + * @param message The exception message. + */ public IngestTaskPipelineException(String message) { super(message); } + /** + * Constructs an exception to be thrown by an ingest task pipeline. + * + * @param message The exception message. + * @param cause The exception cause. + */ public IngestTaskPipelineException(String message, Throwable cause) { super(message, cause); } + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 7c5ab75a69..3341fe9eee 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -125,31 +125,24 @@ final class IngestTasksScheduler { } /** - * Schedules a data source level ingest task and the initial file and result - * ingest tasks for an ingest job. Note that the file filter for the job is - * obtained from the ingest pipeline for the job and that applying the - * filter may cause some or even all of the files in the data source to not - * be scheduled. + * Schedules a data source level ingest task, plus ingest tasks for any + * files and artifacts associated with the data source that are currently in + * the case database. The data source is obtained from the ingest pipeline + * passed in. * - * Note that task scheduling for an ingest job and checking if all of the - * job's tasks are completed must always be atomic operations, enforced by - * synchronization using the scheduler's monitor. Otherwise, the completion - * checks could result in false positives. + * Scheduling these tasks atomically means that it is valid to call + * currentTasksAreCompleted() immediately afterwards. Also note that the + * file filter for the job is obtained from the ingest pipeline and its + * application may cause some or even all of the file tasks to be discarded. * - * There is also a necessary convention that child tasks for products of - * other tasks in the job need to be scheduled before the parent tasks - * notify the scheduler that they are completed. This avoids false positives - * for completion checks by ensuring that the child tasks are queued before - * the parent tasks are marked as completed. - * - * @param ingestPipeline The ingest pipeline for the job. A reference to the - * pipeline is added to each task so that when the - * task is dequeued by an ingest thread and the task's - * execute() method is called, execute() can pass the - * target Content of the task to the pipeline for - * processing by the pipeline's ingest modules. + * @param ingestPipeline The ingest pipeline that will execute the scheduled + * tasks. A reference to the pipeline is added to each + * task so that when the task is dequeued by an ingest + * thread the task can pass the target Content of the + * task to the pipeline for processing by the + * pipeline's ingest modules. */ - synchronized void scheduleDataSourceAndFileIngestTasks(IngestJobPipeline ingestPipeline) { + synchronized void scheduleIngestTasks(IngestJobPipeline ingestPipeline) { if (!ingestPipeline.isCancelled()) { scheduleDataSourceIngestTask(ingestPipeline); scheduleFileIngestTasks(ingestPipeline, Collections.emptyList()); @@ -158,25 +151,15 @@ final class IngestTasksScheduler { } /** - * Schedules a data source level ingest task for an ingest job. + * Schedules a data source level ingest task for an ingest job. The data + * source is obtained from the ingest pipeline passed in. * - * Note that task scheduling for an ingest job and checking if all of the - * job's tasks are completed must always be atomic operations, enforced by - * synchronization using the scheduler's monitor. Otherwise, the completion - * checks could result in false positives. - * - * There is also a necessary convention that child tasks for products of - * other tasks in the job need to be scheduled before the parent tasks - * notify the scheduler that they are completed. This avoids false positives - * for completion checks by ensuring that the child tasks are queued before - * the parent tasks are marked as completed. - * - * @param ingestPipeline The ingest pipeline for the job. A reference to the - * pipeline is added to each task so that when the - * task is dequeued by an ingest thread and the task's - * execute() method is called, execute() can pass the - * target Content of the task to the pipeline for - * processing by the pipeline's ingest modules. + * @param ingestPipeline The ingest pipeline that will execute the scheduled + * task. A reference to the pipeline is added to the + * task so that when the task is dequeued by an ingest + * thread the task can pass the target Content of the + * task to the pipeline for processing by the + * pipeline's ingest modules. */ synchronized void scheduleDataSourceIngestTask(IngestJobPipeline ingestPipeline) { if (!ingestPipeline.isCancelled()) { @@ -192,27 +175,15 @@ final class IngestTasksScheduler { /** * Schedules file tasks for either all the files, or a given subset of the - * files, for an ingest job. Note that the file filter for the job is - * obtained from the ingest pipeline and its application may cause some or - * even all of the files to not be scheduled. + * files, for a data source. The data source is obtained from the ingest + * pipeline passed in. * - * Note that task scheduling for an ingest job and checking if all of the - * job's tasks are completed must always be atomic operations, enforced by - * synchronization using the scheduler's monitor. Otherwise, the completion - * checks could result in false positives. - * - * There is also a necessary convention that child tasks for products of - * other tasks in the job need to be scheduled before the parent tasks - * notify the scheduler that they are completed. This avoids false positives - * for completion checks by ensuring that the child tasks are queued before - * the parent tasks are marked as completed. - * - * @param ingestPipeline The ingest pipeline for the job. A reference to the - * pipeline is added to each task so that when the - * task is dequeued by an ingest thread and the task's - * execute() method is called, execute() can pass the - * target Content of the task to the pipeline for - * processing by the pipeline's ingest modules. + * @param ingestPipeline The ingest pipeline that will execute the scheduled + * tasks. A reference to the pipeline is added to each + * task so that when the task is dequeued by an ingest + * thread the task can pass the target Content of the + * task to the pipeline for processing by the + * pipeline's ingest modules. * @param files A subset of the files from the data source; if * empty, then all if the files from the data source * are candidates for scheduling. @@ -237,20 +208,7 @@ final class IngestTasksScheduler { /** * Schedules file tasks for a collection of "streamed" files for a streaming - * ingest job. Note that the file filter for the job is obtained from the - * ingest pipeline and its application may cause some or even all of the - * files to not be scheduled. - * - * Note that task scheduling for an ingest job and checking if all of the - * job's tasks are completed must always be atomic operations, enforced by - * synchronization using the scheduler's monitor. Otherwise, the completion - * checks could result in false positives. - * - * There is also a necessary convention that child tasks for products of - * other tasks in the job need to be scheduled before the parent tasks - * notify the scheduler that they are completed. This avoids false positives - * for completion checks by ensuring that the child tasks are queued before - * the parent tasks are marked as completed. + * ingest job. * * @param ingestPipeline The ingest pipeline for the job. A reference to the * pipeline is added to each task so that when the @@ -284,17 +242,6 @@ final class IngestTasksScheduler { * be used to schedule files that are products of ingest module processing, * e.g., extracted files and carved files. * - * Note that task scheduling for an ingest job and checking if all of the - * job's tasks are completed must always be atomic operations, enforced by - * synchronization using the scheduler's monitor. Otherwise, the completion - * checks could result in false positives. - * - * There is also a necessary convention that child tasks for products of - * other tasks in the job need to be scheduled before the parent tasks - * notify the scheduler that they are completed. This avoids false positives - * for completion checks by ensuring that the child tasks are queued before - * the parent tasks are marked as completed. - * * @param ingestPipeline The ingest pipeline for the job. A reference to the * pipeline is added to each task so that when the * task is dequeued by an ingest thread and the task's @@ -330,19 +277,9 @@ final class IngestTasksScheduler { } /** - * Schedules data artifact ingest tasks for an ingest job for the data - * artifacts that have already been added to the case database. - * - * Note that task scheduling for an ingest job and checking if all of the - * job's tasks are completed must always be atomic operations, enforced by - * synchronization using the scheduler's monitor. Otherwise, the completion - * checks could result in false positives. - * - * There is also a necessary convention that child tasks for products of - * other tasks in the job need to be scheduled before the parent tasks - * notify the scheduler that they are completed. This avoids false positives - * for completion checks by ensuring that the child tasks are queued before - * the parent tasks are marked as completed. + * Schedules data artifact ingest tasks for any data artifacts that have + * already been added to the case database for a data source. The data + * source is obtained from the ingest pipeline passed in. * * @param ingestPipeline The ingest pipeline for the job. A reference to the * pipeline is added to each task so that when the @@ -369,17 +306,6 @@ final class IngestTasksScheduler { * intended to be used to schedule artifacts that are products of ingest * module processing. * - * Note that task scheduling for an ingest job and checking if all of the - * job's tasks are completed must always be atomic operations, enforced by - * synchronization using the scheduler's monitor. Otherwise, the completion - * checks could result in false positives. - * - * There is also a necessary convention that child tasks for products of - * other tasks in the job need to be scheduled before the parent tasks - * notify the scheduler that they are completed. This avoids false positives - * for completion checks by ensuring that the child tasks are queued before - * the parent tasks are marked as completed. - * * @param ingestPipeline The ingest pipeline for the job. A reference to the * pipeline is added to each task so that when the * task is dequeued by an ingest thread and the task's @@ -619,7 +545,7 @@ final class IngestTasksScheduler { for (Content child : file.getChildren()) { if (child instanceof AbstractFile) { AbstractFile childFile = (AbstractFile) child; - FileIngestTask childTask = new FileIngestTask(nextTask.getIngestPipeline(), childFile); + FileIngestTask childTask = new FileIngestTask(nextTask.getIngestJobPipeline(), childFile); if (childFile.hasChildren()) { batchedFileIngestTasksQueue.add(childTask); } else if (shouldEnqueueFileTask(childTask)) { @@ -738,7 +664,7 @@ final class IngestTasksScheduler { private static boolean shouldBeCarved(final FileIngestTask task) { try { AbstractFile file = task.getFile(); - return task.getIngestPipeline().shouldProcessUnallocatedSpace() && file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS); + return task.getIngestJobPipeline().shouldProcessUnallocatedSpace() && file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS); } catch (TskCoreException ex) { return false; } @@ -755,7 +681,7 @@ final class IngestTasksScheduler { private static boolean fileAcceptedByFilter(final FileIngestTask task) { try { AbstractFile file = task.getFile(); - return !(task.getIngestPipeline().getFileIngestFilter().fileIsMemberOf(file) == null); + return !(task.getIngestJobPipeline().getFileIngestFilter().fileIsMemberOf(file) == null); } catch (TskCoreException ex) { return false; } @@ -772,7 +698,7 @@ final class IngestTasksScheduler { */ synchronized private static boolean hasTasksForJob(Collection tasks, long pipelineId) { for (IngestTask task : tasks) { - if (task.getIngestPipeline().getId() == pipelineId) { + if (task.getIngestJobPipeline().getId() == pipelineId) { return true; } } @@ -790,7 +716,7 @@ final class IngestTasksScheduler { Iterator iterator = tasks.iterator(); while (iterator.hasNext()) { IngestTask task = iterator.next(); - if (task.getIngestPipeline().getId() == pipelineId) { + if (task.getIngestJobPipeline().getId() == pipelineId) { iterator.remove(); } } @@ -808,7 +734,7 @@ final class IngestTasksScheduler { private static int countTasksForJob(Collection tasks, long pipelineId) { int count = 0; for (IngestTask task : tasks) { - if (task.getIngestPipeline().getId() == pipelineId) { + if (task.getIngestJobPipeline().getId() == pipelineId) { count++; } } From 2d7052fa29113b9171839ade6e004bbb75e5d022 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 28 May 2021 12:25:55 -0400 Subject: [PATCH 09/63] Atrifact ingest pipeline work --- .../CentralRepoDataArtifactIngestModule.java | 9 +++++-- .../ingest/DataArtifactIngestPipeline.java | 19 +++++++------ .../ingest/DataArtifactIngestTask.java | 6 +++++ .../autopsy/ingest/DataSourceIngestTask.java | 6 +++++ .../autopsy/ingest/FileIngestTask.java | 6 +++++ .../autopsy/ingest/IngestJobPipeline.java | 27 +++---------------- .../sleuthkit/autopsy/ingest/IngestTask.java | 24 +++++++++++------ 7 files changed, 53 insertions(+), 44 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java index 9820307ecd..0ed9f53518 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java @@ -25,8 +25,13 @@ import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.DataArtifact; /** - * A placeholder data artifact ingest module that counts the number of data - * artifacts it processes and posts the final count to the ingest inbox. + * RJCTODO + * + * NOTE TO REVIEWER: + * + * This is a placeholder data artifact ingest module that counts the number of + * data artifacts it processes and posts the final count to the ingest inbox. + * The guts of the module will be supplied by a later PR. */ public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestModule { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java index 5b6ca85120..af6cc037a4 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java @@ -23,16 +23,18 @@ import java.util.Optional; import org.sleuthkit.datamodel.DataArtifact; /** - * A pipeline of data artifact ingest modules for executing data artifact + * A pipeline of data artifact ingest modules used to execute data artifact * ingest tasks for an ingest job. */ final class DataArtifactIngestPipeline extends IngestTaskPipeline { /** - * Constructs a pipeline of data artifact ingest modules for executing data - * artifact ingest tasks for an ingest job. + * Constructs a pipeline of data artifact ingest modules used to execute + * data artifact ingest tasks for an ingest job. * - * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. + * + * @param ingestJobPipeline The ingest job pipeline that owns this ingest + * task pipeline. * @param moduleTemplates The ingest module templates that define this * pipeline. May be an empty list. */ @@ -67,8 +69,8 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline errors = new ArrayList<>(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index 5c4b4148b4..41c3736986 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -27,7 +27,7 @@ import org.sleuthkit.datamodel.Content; * that analyze data artifacts. */ abstract class IngestTask { - + private final static long NOT_SET = Long.MIN_VALUE; private final IngestJobPipeline ingestJobPipeline; private long threadId; @@ -73,15 +73,23 @@ abstract class IngestTask { return threadId; } + /** + * Sets the thread ID of the ingest thread executing this task. + * + * @param threadId The thread ID. + */ + void setThreadId(long threadId) { + this.threadId = threadId; + } + /** * Records the ingest thread ID of the calling thread and executes this task - * using the ingest job pipeline specified when the task was created. + * using the ingest job pipeline specified when the task was created. The + * implementation of the method should simple call + * super.setThreadId(threadId) and getIngestJobPipeline().process(this). * * @param threadId The numeric ID of the ingest thread executing this task. -= */ - void execute(long threadId) { - this.threadId = threadId; - ingestJobPipeline.execute(this); - } - + */ + abstract void execute(long threadId); + } From 1c7e3a3b55931a84797ecbc7654c84df5dd36b21 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 28 May 2021 12:54:54 -0400 Subject: [PATCH 10/63] Atrifact ingest pipeline work --- .../ingest/DataArtifactIngestPipeline.java | 1 - .../autopsy/ingest/FileIngestTask.java | 1 - .../sleuthkit/autopsy/ingest/IngestJob.java | 11 +----- .../autopsy/ingest/IngestJobContext.java | 38 +++++++++---------- .../autopsy/ingest/IngestJobPipeline.java | 4 +- 5 files changed, 21 insertions(+), 34 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java index af6cc037a4..824d7d7fe9 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java @@ -32,7 +32,6 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline files, IngestJobSettings settings) { + IngestJob(DataSource dataSource, List files, IngestJobSettings settings) { this(Arrays.asList(dataSource), settings); this.files.addAll(files); } @@ -137,14 +138,6 @@ public final class IngestJob { public long getId() { return this.id; } - - /** - * RJCTODO - * @return - */ - List getFiles() { - return files; - } /** * Checks to see if this ingest job has at least one non-empty ingest module diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 62b531e541..aec1c3a737 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -18,18 +18,14 @@ */ package org.sleuthkit.autopsy.ingest; -import java.util.ArrayList; import java.util.List; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataArtifact; /** - * An ingest job context object that provides an ingest module with operations - * specific to the ingest job of which the module is a part. - * - * From a coding perspective, this object provides an abstraction that hides - * ingest infrastructure details. + * Provides an ingest module with services specific to the ingest job of which + * the module is a part. */ public final class IngestJobContext { @@ -37,7 +33,7 @@ public final class IngestJobContext { /** * Constructs an ingest job context object that provides an ingest module - * with operations specific to the ingest job of which the module is a part. + * with services specific to the ingest job of which the module is a part.. * * @param ingestJobPipeline */ @@ -72,6 +68,20 @@ public final class IngestJobContext { return ingestPipeline.getId(); } + /** + * Queries whether or not cancellation of the data source ingest part of the + * ingest job associated with this context has been requested. + * + * @return True or false. + * + * @deprecated Use dataSourceIngestIsCancelled() or fileIngestIsCancelled() + * instead. + */ + @Deprecated + public boolean isJobCancelled() { + return dataSourceIngestIsCancelled(); + } + /** * Checks whether or not cancellation of the currently running data source * level ingest module for the ingest job has been requested. Data source @@ -161,18 +171,4 @@ public final class IngestJobContext { addFilesToJob(files); } - /** - * Queries whether or not cancellation of the data source ingest part of the - * ingest job associated with this context has been requested. - * - * @return True or false. - * - * @deprecated Use dataSourceIngestIsCancelled() or fileIngestIsCancelled() - * instead. - */ - @Deprecated - public boolean isJobCancelled() { - return dataSourceIngestIsCancelled(); - } - } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 95a0a5b278..5afedf28c8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -1186,8 +1186,8 @@ final class IngestJobPipeline { fileIngestPipelinesQueue.put(pipeline); } } catch (InterruptedException ex) { - // RJCTODO This probablly should be logged, interrupt during wait for pipeline copy - // Also need to reset the flag... + logger.log(Level.SEVERE, String.format("Unexpected interrupt of file ingest thread during execution of file ingest job (file obj ID = %d)", task.getFileId()), ex); + Thread.currentThread().interrupt(); // Reset thread interrupted flag } finally { taskScheduler.notifyTaskCompleted(task); checkForStageCompleted(); From 1c2241285ba5121fc7435854678b56b3d2102934 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 28 May 2021 14:55:15 -0400 Subject: [PATCH 11/63] Atrifact ingest pipeline work --- .../sleuthkit/autopsy/ingest/IngestJob.java | 3 +- .../autopsy/ingest/IngestJobContext.java | 28 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 40b14942da..a8024a3252 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -33,7 +33,6 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.DataSource; /** * Analyzes one or more data sources using a set of ingest modules specified via @@ -109,7 +108,7 @@ public final class IngestJob { * @param files A subset of the files for the data source. * @param settings The ingest job settings. */ - IngestJob(DataSource dataSource, List files, IngestJobSettings settings) { + IngestJob(Content dataSource, List files, IngestJobSettings settings) { this(Arrays.asList(dataSource), settings); this.files.addAll(files); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index aec1c3a737..1766ad3d7b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -29,16 +29,16 @@ import org.sleuthkit.datamodel.DataArtifact; */ public final class IngestJobContext { - private final IngestJobPipeline ingestPipeline; + private final IngestJobPipeline ingestJobPipeline; /** * Constructs an ingest job context object that provides an ingest module * with services specific to the ingest job of which the module is a part.. * - * @param ingestJobPipeline + * @param ingestJobPipeline The ingest pipeline for the job. */ IngestJobContext(IngestJobPipeline ingestJobPipeline) { - this.ingestPipeline = ingestJobPipeline; + this.ingestJobPipeline = ingestJobPipeline; } /** @@ -47,7 +47,7 @@ public final class IngestJobContext { * @return The context string. */ public String getExecutionContext() { - return ingestPipeline.getExecutionContext(); + return ingestJobPipeline.getExecutionContext(); } /** @@ -56,7 +56,7 @@ public final class IngestJobContext { * @return The data source. */ public Content getDataSource() { - return ingestPipeline.getDataSource(); + return ingestJobPipeline.getDataSource(); } /** @@ -65,7 +65,7 @@ public final class IngestJobContext { * @return The ID. */ public long getJobId() { - return ingestPipeline.getId(); + return ingestJobPipeline.getId(); } /** @@ -80,8 +80,8 @@ public final class IngestJobContext { @Deprecated public boolean isJobCancelled() { return dataSourceIngestIsCancelled(); - } - + } + /** * Checks whether or not cancellation of the currently running data source * level ingest module for the ingest job has been requested. Data source @@ -91,7 +91,7 @@ public final class IngestJobContext { * @return True or false. */ public boolean dataSourceIngestIsCancelled() { - return ingestPipeline.currentDataSourceIngestModuleIsCancelled() || ingestPipeline.isCancelled(); + return ingestJobPipeline.currentDataSourceIngestModuleIsCancelled() || ingestJobPipeline.isCancelled(); } /** @@ -108,7 +108,7 @@ public final class IngestJobContext { * modules. File ingest cancellation is equiovalent to ingest job * cancellation. */ - return ingestPipeline.isCancelled(); + return ingestJobPipeline.isCancelled(); } /** @@ -125,7 +125,7 @@ public final class IngestJobContext { * modules. Data artifact ingest cancellation is equivalent to ingest * job cancellation. */ - return ingestPipeline.isCancelled(); + return ingestJobPipeline.isCancelled(); } /** @@ -135,7 +135,7 @@ public final class IngestJobContext { * @return True or false. */ public boolean processingUnallocatedSpace() { - return ingestPipeline.shouldProcessUnallocatedSpace(); + return ingestJobPipeline.shouldProcessUnallocatedSpace(); } /** @@ -145,7 +145,7 @@ public final class IngestJobContext { * @param files The files. */ public void addFilesToJob(List files) { - ingestPipeline.addFiles(files); + ingestJobPipeline.addFiles(files); } /** @@ -155,7 +155,7 @@ public final class IngestJobContext { * @param artifacts The artifacts. */ public void addDataArtifactsToJob(List artifacts) { - ingestPipeline.addDataArtifacts(artifacts); + ingestJobPipeline.addDataArtifacts(artifacts); } /** From 2ba9a67dbee243be458a39cd0f266022c2bca13e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 28 May 2021 15:09:35 -0400 Subject: [PATCH 12/63] Atrifact ingest pipeline work --- .../autopsy/ingest/IngestJobContext.java | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 1766ad3d7b..84e9db0f08 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -69,24 +69,20 @@ public final class IngestJobContext { } /** - * Queries whether or not cancellation of the data source ingest part of the - * ingest job associated with this context has been requested. + * Indicates whether or not cancellation of the ingest job has been + * requested. * * @return True or false. - * - * @deprecated Use dataSourceIngestIsCancelled() or fileIngestIsCancelled() - * instead. */ - @Deprecated public boolean isJobCancelled() { - return dataSourceIngestIsCancelled(); + return ingestJobPipeline.isCancelled(); } /** - * Checks whether or not cancellation of the currently running data source - * level ingest module for the ingest job has been requested. Data source - * level ingest modules should check this periodically and break off - * processing if the method returns true. + * Indicates whether or not cancellation of the currently running data + * source level ingest module has been requested. Data source level ingest + * modules should check this periodically and break off processing if the + * method returns true. * * @return True or false. */ @@ -95,18 +91,16 @@ public final class IngestJobContext { } /** - * Checks whether or not cancellation of the currently running file ingest - * module for the ingest job has been requested. File ingest modules should - * check this periodically and break off processing if the method returns - * true. + * Indicates whether or not cancellation of the currently running file level + * ingest module has been requested. File level ingest modules should check + * this periodically and break off processing if the method returns true. * * @return True or false. */ public boolean fileIngestIsCancelled() { /* * It is not currently possible to cancel individual file ingest - * modules. File ingest cancellation is equiovalent to ingest job - * cancellation. + * modules. */ return ingestJobPipeline.isCancelled(); } @@ -121,9 +115,8 @@ public final class IngestJobContext { */ public boolean dataArtifactIngestIsCancelled() { /* - * It is not currently possible to cancel individual file ingest - * modules. Data artifact ingest cancellation is equivalent to ingest - * job cancellation. + * It is not currently possible to cancel individual data artifact + * ingest modules. */ return ingestJobPipeline.isCancelled(); } From d34a5860cf6370990c171e44caa19d2e850ad506 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 28 May 2021 15:19:56 -0400 Subject: [PATCH 13/63] Artifact ingest pipeline work --- .../autopsy/ingest/IngestJobContext.java | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 84e9db0f08..6f1724d0b7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -33,7 +33,7 @@ public final class IngestJobContext { /** * Constructs an ingest job context object that provides an ingest module - * with services specific to the ingest job of which the module is a part.. + * with services specific to the ingest job of which the module is a part. * * @param ingestJobPipeline The ingest pipeline for the job. */ @@ -73,7 +73,11 @@ public final class IngestJobContext { * requested. * * @return True or false. + * + * @deprecated Modules should call a type-specific cancellation check method + * instead. */ + @Deprecated public boolean isJobCancelled() { return ingestJobPipeline.isCancelled(); } @@ -107,9 +111,9 @@ public final class IngestJobContext { /** * Checks whether or not cancellation of the currently running data artifact - * ingest module for the ingest job has been requested. File ingest modules - * should check this periodically and break off processing if the method - * returns true. + * ingest module for the ingest job has been requested. Data artifact ingest + * modules should check this periodically and break off processing if the + * method returns true. * * @return True or false. */ @@ -131,6 +135,19 @@ public final class IngestJobContext { return ingestJobPipeline.shouldProcessUnallocatedSpace(); } + /** + * Adds one or more files, i.e., extracted or carved files, to the ingest + * job associated with this context. + * + * @param files The files to be added. + * + * @deprecated use addFilesToJob() instead. + */ + @Deprecated + public void scheduleFiles(List files) { + addFilesToJob(files); + } + /** * Adds one or more files, e.g., extracted or carved files, to the ingest * job for processing by its file ingest modules. @@ -151,17 +168,4 @@ public final class IngestJobContext { ingestJobPipeline.addDataArtifacts(artifacts); } - /** - * Adds one or more files, i.e., extracted or carved files, to the ingest - * job associated with this context. - * - * @param files The files to be added. - * - * @deprecated use addFilesToJob() instead. - */ - @Deprecated - public void scheduleFiles(List files) { - addFilesToJob(files); - } - } From 13d15d51efb26189198ebbf758c7925b888b00ee Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 2 Jun 2021 09:44:08 -0400 Subject: [PATCH 14/63] 7332 artifact pipeline work --- .../sleuthkit/autopsy/ingest/IngestJob.java | 2 +- .../autopsy/ingest/IngestJobPipeline.java | 208 +++++++----------- .../autopsy/ingest/IngestTasksScheduler.java | 30 +-- 3 files changed, 101 insertions(+), 139 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index a8024a3252..a82c3878b0 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -174,7 +174,7 @@ public final class IngestJob { } // Streaming ingest jobs will only have one data source IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); - streamingIngestPipeline.addStreamingIngestDataSource(); + streamingIngestPipeline.notifyFileStreamingCompleted(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 5afedf28c8..2c2a6731f8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -72,7 +72,7 @@ final class IngestJobPipeline { /* * A regular expression for identifying the proxy classes Jython generates - * for ingest module factories classes written using Python. For example: + * for ingest module factories written using Python. For example: * org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14 */ private static final Pattern JYTHON_MODULE_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); @@ -81,7 +81,7 @@ final class IngestJobPipeline { * These fields define an ingest pipeline: the parent ingest job, a pipeline * ID, the user's ingest job settings, and the data source to be analyzed. * Optionally, there is a set of files to be analyzed, instead of analyzing - * ALL of the files in the data source. + * all of the files in the data source. * * The pipeline ID is used to associate the pipeline with its ingest tasks. * The ingest job ID cannot be used for this purpose because the parent @@ -126,21 +126,19 @@ final class IngestJobPipeline { */ FINALIZATION }; - @GuardedBy("stageTransitionLock") - private Stages stage = IngestJobPipeline.Stages.INITIALIZATION; + private volatile Stages stage = IngestJobPipeline.Stages.INITIALIZATION; private final Object stageTransitionLock = new Object(); - /** + /* * An ingest pipeline has separate data source level ingest task pipelines * for the first and second stages. Longer running, lower priority modules * belong in the second stage pipeline. */ - private final Object dataSourceIngestPipelineLock = new Object(); private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; - private DataSourceIngestPipeline currentDataSourceIngestPipeline; + private volatile DataSourceIngestPipeline currentDataSourceIngestPipeline; - /** + /* * An ingest pipeline has a collection of identical file ingest task * pipelines, one for each file ingest thread in the ingest manager. The * ingest threads take ingest task pipelines as they need them and return @@ -156,9 +154,9 @@ final class IngestJobPipeline { */ private DataArtifactIngestPipeline artifactIngestPipeline; - /** + /* * An ingest pipeline supports cancellation of just its currently running - * data source level ingest task pipeline or cancellation of ALL of its + * data source level ingest task pipeline or cancellation of all of its * child ingest task pipelines. Cancellation works by setting flags that are * checked by the ingest task pipelines every time they transition from one * module to another. Modules are also expected to check these flags (via @@ -174,15 +172,11 @@ final class IngestJobPipeline { /* * An ingest pipeline interacts with the ingest task scheduler to create and * queue ingest tasks and to determine whether or not there are ingest tasks - * still to be executed so that the pipeline can transition through its - * stages. The ingest modules in the pipeline can schedule ingest tasks as - * well (via the ingest job context). For example, a file carving module can - * add carved files to the ingest job and most modules will add data - * artifacts to the ingest job. + * still to be executed. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - /** + /* * If running in a GUI, an ingest pipeline reports progress and allows a * user to cancel either an individual data source level ingest module or * all of its ingest tasks using progress bars in the lower right hand @@ -200,6 +194,11 @@ final class IngestJobPipeline { private ProgressHandle fileIngestProgressBar; private final Object artifactIngestProgressLock = new Object(); private ProgressHandle artifactIngestProgressBar; + + /* + * Ingest pipeline details are tracked using this object and are recorded in + * the case database. + */ private volatile IngestJobInfo ingestJobInfo; /** @@ -269,10 +268,12 @@ final class IngestJobPipeline { * first and third party modules next. * * @param orderedModules The list to populate. - * @param javaModules The input ingest module templates for modules - * implemented using Java. - * @param jythonModules The input ingest module templates for modules - * implemented using Jython. + * @param javaModules A map of the input ingest module templates for + * modules implemented using Java, keyed by + * fully-qualified (canonical) class name. + * @param jythonModules A map of the input ingest module templates for + * modules implemented using Jython, keyed by + * fully-qualified (canonical) class name. */ private static void completePipeline(final List orderedModules, final Map javaModules, final Map jythonModules) { final List autopsyModules = new ArrayList<>(); @@ -349,8 +350,8 @@ final class IngestJobPipeline { * Sort the ingest module templates into buckets based on the module * types the ingest module factory can create. A template may go into * more than one bucket. The buckets are actually maps of ingest module - * factory class names to ingest module templates. The maps are used to - * go from an ingest module factory class name read from the pipeline + * factory class names to ingest module templates. These maps are used + * to go from an ingest module factory class name read from the pipeline * configuration file to the corresponding ingest module template. * * There are actually two maps for each module type bucket. One map is @@ -430,7 +431,7 @@ final class IngestJobPipeline { * module templates. * @param pipelineConfig An ordered list of ingest module * factory class names representing an - * ingest pipeline, read form the + * ingest pipeline, read from the * pipeline configuration file. * * @return An ordered list of ingest module templates, i.e., an @@ -457,15 +458,6 @@ final class IngestJobPipeline { return pipelineId; } - /** - * Gets the parent ingest job of this ingest pipeline. - * - * @return The ingest job. - */ - IngestJob getIngestJob() { - return job; - } - /** * Gets the ingest execution context name. * @@ -484,16 +476,6 @@ final class IngestJobPipeline { return dataSource; } - /** - * Gets the subset of the files from the data source to be analyzed by this - * ingest pipeline. - * - * @return The files. - */ - List getFiles() { - return Collections.unmodifiableList(files); - } - /** * Queries whether or not unallocated space should be processed by this * ingest pipeline. @@ -523,32 +505,46 @@ final class IngestJobPipeline { return hasFileIngestModules() || hasFirstStageDataSourceIngestModules() || hasSecondStageDataSourceIngestModules() - || hasArtifactIngestModules(); + || hasDataArtifactIngestModules(); + } + + /** + * Checks to see if this ingest pipeline has at least one ingest module to + * run. + * + * @return True or false. + */ + boolean hasDataSourceIngestModules() { + if (stage == Stages.SECOND_STAGE) { + return hasSecondStageDataSourceIngestModules(); + } else { + return hasFirstStageDataSourceIngestModules(); + } } /** * Checks to see if this ingest pipeline has at least one first stage data - * source level ingest modules. + * source level ingest module to run. * * @return True or false. */ - boolean hasFirstStageDataSourceIngestModules() { + private boolean hasFirstStageDataSourceIngestModules() { return (firstStageDataSourceIngestPipeline.isEmpty() == false); } /** * Checks to see if this ingest pipeline has at least one second stage data - * source level ingest module. + * source level ingest module to run. * * @return True or false. */ - boolean hasSecondStageDataSourceIngestModules() { + private boolean hasSecondStageDataSourceIngestModules() { return (secondStageDataSourceIngestPipeline.isEmpty() == false); } /** - * Checks to see if this ingest pipeline has at least one file ingest - * module. + * Checks to see if this ingest pipeline has at least one file ingest module + * to run. * * @return True or false. */ @@ -563,12 +559,12 @@ final class IngestJobPipeline { } /** - * Checks to see if this ingest pipeline has at least one artifact ingest - * module. + * Checks to see if this ingest pipeline has at least one data artifact + * ingest module to run. * * @return True or false. */ - boolean hasArtifactIngestModules() { + boolean hasDataArtifactIngestModules() { return (artifactIngestPipeline.isEmpty() == false); } @@ -581,7 +577,7 @@ final class IngestJobPipeline { List errors = startUpIngestTaskPipelines(); if (errors.isEmpty()) { recordIngestJobStartUpInfo(); - if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasArtifactIngestModules()) { + if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { if (job.getIngestMode() == IngestJob.Mode.STREAMING) { startFirstStageInStreamingMode(); } else { @@ -701,13 +697,12 @@ final class IngestJobPipeline { * already been added to the case database by the data source processor. */ private void startFirstStage() { + /* + * Do a count of the files the data source processor has added to the + * case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a GUI. + */ if (hasFileIngestModules()) { - /* - * Do a count of the files the data source processor has added to - * the case database. This estimate will be used for ingest progress - * snapshots and for the file ingest progress bar if running with a - * GUI. - */ long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; synchronized (fileIngestProgressLock) { estimatedFilesToProcess = filesToProcess; @@ -719,28 +714,21 @@ final class IngestJobPipeline { * hand corner of the main application window. */ if (doUI) { - if (hasFirstStageDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } if (hasFileIngestModules()) { startFileIngestProgressBar(); } - if (hasArtifactIngestModules()) { + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + if (hasDataArtifactIngestModules()) { startArtifactIngestProgressBar(); } } - /* - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (dataSourceIngestPipelineLock) { - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - } - synchronized (stageTransitionLock) { logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS stage = Stages.FIRST_STAGE; + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; /** * Schedule the first stage ingest tasks and then immediately check @@ -751,7 +739,11 @@ final class IngestJobPipeline { * executing an ingest task, so such a job would run forever without * the check here. */ - taskScheduler.scheduleIngestTasks(this); + if (!files.isEmpty() && hasFileIngestModules()) { + taskScheduler.scheduleFileIngestTasks(this, files); + } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { + taskScheduler.scheduleIngestTasks(this); + } checkForStageCompleted(); } } @@ -764,31 +756,21 @@ final class IngestJobPipeline { * addStreamingIngestFiles() and addStreamingIngestDataSource(). */ private void startFirstStageInStreamingMode() { - if (hasFileIngestModules()) { - synchronized (fileIngestProgressLock) { - /* - * Start with zero to signal an unknown value. This estimate - * will be used for ingest progress snapshots and for the file - * ingest progress bar if running with a GUI. - */ - estimatedFilesToProcess = 0; - } - } - - /* - * If running with a GUI, start ingest progress bars in the lower right - * hand corner of the main application window. - */ if (doUI) { + /* + * If running with a GUI, start ingest progress bars in the lower + * right hand corner of the main application window. + */ if (hasFileIngestModules()) { /* - * Note that because estimated files remaining to process has - * been set to zero, the progress bar will start in the + * Note that because estimated files remaining to process still + * has its initial value of zero since files are still be added + * to the case database, the progress bar will start in the * "indeterminate" state. */ startFileIngestProgressBar(); } - if (hasArtifactIngestModules()) { + if (hasDataArtifactIngestModules()) { startArtifactIngestProgressBar(); } } @@ -796,7 +778,7 @@ final class IngestJobPipeline { synchronized (stageTransitionLock) { logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS stage = Stages.FIRST_STAGE_STREAMING; - if (hasArtifactIngestModules()) { + if (hasDataArtifactIngestModules()) { /* * Schedule artifact ingest tasks for any artifacts currently in * the case database. This needs to be done before any files or @@ -813,7 +795,7 @@ final class IngestJobPipeline { * Start data source ingest. Used for streaming ingest when the data source * is not ready when ingest starts. */ - void addStreamingIngestDataSource() { + void notifyFileStreamingCompleted() { /* * Do a count of the files the data source processor has added to the * case database. This estimate will be used for ingest progress @@ -836,17 +818,11 @@ final class IngestJobPipeline { } } - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - synchronized (stageTransitionLock) { logInfoMessage("Adding the data source in streaming mode"); //NON-NLS stage = IngestJobPipeline.Stages.FIRST_STAGE; + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + if (hasFirstStageDataSourceIngestModules()) { IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); } else { @@ -870,12 +846,10 @@ final class IngestJobPipeline { if (doUI) { startDataSourceIngestProgressBar(); } - synchronized (dataSourceIngestPipelineLock) { - currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; - } synchronized (stageTransitionLock) { logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), job.getId())); //NON-NLS stage = IngestJobPipeline.Stages.SECOND_STAGE; + currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; taskScheduler.scheduleDataSourceIngestTask(this); } } @@ -1087,29 +1061,13 @@ final class IngestJobPipeline { */ void execute(DataSourceIngestTask task) { try { - synchronized (dataSourceIngestPipelineLock) { - if (!isCancelled() && !currentDataSourceIngestPipeline.isEmpty()) { - List errors = new ArrayList<>(); - errors.addAll(currentDataSourceIngestPipeline.executeTask(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } + if (!isCancelled()) { + List errors = new ArrayList<>(); + errors.addAll(currentDataSourceIngestPipeline.executeTask(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); } } - - if (doUI) { - /** - * Shut down the data source ingest progress bar right away. - * Data source-level processing is finished for this stage. - */ - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; - } - } - } - } finally { taskScheduler.notifyTaskCompleted(task); checkForStageCompleted(); @@ -1425,7 +1383,7 @@ final class IngestJobPipeline { * @return The currently running module, may be null. */ DataSourceIngestPipeline.DataSourcePipelineModule getCurrentDataSourceIngestModule() { - if (null != currentDataSourceIngestPipeline) { + if (currentDataSourceIngestPipeline != null) { return (DataSourceIngestPipeline.DataSourcePipelineModule) currentDataSourceIngestPipeline.getCurrentlyRunningModule(); } else { return null; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 3341fe9eee..530ff1845b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -125,15 +125,11 @@ final class IngestTasksScheduler { } /** - * Schedules a data source level ingest task, plus ingest tasks for any - * files and artifacts associated with the data source that are currently in - * the case database. The data source is obtained from the ingest pipeline - * passed in. - * - * Scheduling these tasks atomically means that it is valid to call - * currentTasksAreCompleted() immediately afterwards. Also note that the - * file filter for the job is obtained from the ingest pipeline and its - * application may cause some or even all of the file tasks to be discarded. + * Schedules ingest tasks based on the types of ingest modules that the + * ingest pipeline that will exedute tasks has. Scheduling these tasks + * atomically means that it is valid to call currentTasksAreCompleted() + * immediately after calling this method. Note that the may cause some or + * even all of any file tasks to be discarded. * * @param ingestPipeline The ingest pipeline that will execute the scheduled * tasks. A reference to the pipeline is added to each @@ -144,9 +140,15 @@ final class IngestTasksScheduler { */ synchronized void scheduleIngestTasks(IngestJobPipeline ingestPipeline) { if (!ingestPipeline.isCancelled()) { - scheduleDataSourceIngestTask(ingestPipeline); - scheduleFileIngestTasks(ingestPipeline, Collections.emptyList()); - scheduleDataArtifactIngestTasks(ingestPipeline); + if (ingestPipeline.hasDataSourceIngestModules()) { + scheduleDataSourceIngestTask(ingestPipeline); + } + if (ingestPipeline.hasFileIngestModules()) { + scheduleFileIngestTasks(ingestPipeline, Collections.emptyList()); + } + if (ingestPipeline.hasDataArtifactIngestModules()) { + scheduleDataArtifactIngestTasks(ingestPipeline); + } } } @@ -312,7 +314,9 @@ final class IngestTasksScheduler { * execute() method is called, execute() can pass the * target Content of the task to the pipeline for * processing by the pipeline's ingest modules. - * @param artifacts The artifacts. + * @param artifacts A subset of the data artifacts from the data + * source; if empty, then all of the data artifacts + * from the data source will be scheduled. */ synchronized void scheduleDataArtifactIngestTasks(IngestJobPipeline ingestPipeline, List artifacts) { if (!ingestPipeline.isCancelled()) { From 7b9bc62f592925448e7cce307206e480dc2b19b4 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 3 Jun 2021 06:31:40 -0400 Subject: [PATCH 15/63] 7332 artifact pipeline --- .../autopsy/ingest/Bundle.properties | 1 + .../autopsy/ingest/Bundle.properties-MERGED | 1 + .../autopsy/ingest/IngestJobPipeline.java | 254 ++++++++---------- .../autopsy/ingest/IngestManager.java | 2 +- .../ingest/IngestProgressSnapshotPanel.java | 20 +- .../autopsy/ingest/IngestTasksScheduler.java | 36 ++- .../sleuthkit/autopsy/ingest/Snapshot.java | 7 + .../pictureanalyzer/impls/EXIFProcessor.java | 17 +- 8 files changed, 175 insertions(+), 163 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index 978e361baa..98959b35b5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -92,6 +92,7 @@ IngestJobTableModel.colName.dirQueued=Dir Queued IngestJobTableModel.colName.rootQueued=Root Queued IngestJobTableModel.colName.streamingQueued=Streaming Queued IngestJobTableModel.colName.dsQueued=DS Queued +IngestJobTableModel.colName.artifactsQueued=Artifacts Queued ModuleTableModel.colName.module=Module ModuleTableModel.colName.duration=Duration IngestJobSettingsPanel.jButtonSelectAll.text=Select All diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED index 70c273222d..f852f1c3f3 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED @@ -108,6 +108,7 @@ IngestJobTableModel.colName.dirQueued=Dir Queued IngestJobTableModel.colName.rootQueued=Root Queued IngestJobTableModel.colName.streamingQueued=Streaming Queued IngestJobTableModel.colName.dsQueued=DS Queued +IngestJobTableModel.colName.artifactsQueued=Artifacts Queued ModuleTableModel.colName.module=Module ModuleTableModel.colName.duration=Duration IngestJobSettingsPanel.jButtonSelectAll.text=Select All diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 2c2a6731f8..0af30ab5d9 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -31,7 +31,6 @@ import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; -import javax.annotation.concurrent.GuardedBy; import javax.swing.JOptionPane; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; @@ -109,7 +108,7 @@ final class IngestJobPipeline { * by a data source processor. The data source has not been added to the * pipeline yet. */ - FIRST_STAGE_STREAMING, + FIRST_STAGE_FILES_ONLY, /* * The pipeline is running one or more of the following three types of * ingest modules: higher priority data source level ingest modules, @@ -579,7 +578,7 @@ final class IngestJobPipeline { recordIngestJobStartUpInfo(); if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { if (job.getIngestMode() == IngestJob.Mode.STREAMING) { - startFirstStageInStreamingMode(); + startFirstStageFilesOnly(); } else { startFirstStage(); } @@ -725,27 +724,24 @@ final class IngestJobPipeline { } } - synchronized (stageTransitionLock) { - logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS - stage = Stages.FIRST_STAGE; - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS + stage = Stages.FIRST_STAGE; + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - /** - * Schedule the first stage ingest tasks and then immediately check - * for stage completion. This is necessary because it is possible - * that zero tasks will actually make it to task execution due to - * the file filter or other ingest job settings. In that case, there - * will never be a stage completion check in an ingest thread - * executing an ingest task, so such a job would run forever without - * the check here. - */ - if (!files.isEmpty() && hasFileIngestModules()) { - taskScheduler.scheduleFileIngestTasks(this, files); - } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { - taskScheduler.scheduleIngestTasks(this); - } - checkForStageCompleted(); + /** + * Schedule the first stage ingest tasks and then immediately check for + * stage completion. This is necessary because it is possible that zero + * tasks will actually make it to task execution due to the file filter + * or other ingest job settings. In that case, there will never be a + * stage completion check in an ingest thread executing an ingest task, + * so such a job would run forever without the check here. + */ + if (!files.isEmpty() && hasFileIngestModules()) { + taskScheduler.scheduleFileIngestTasks(this, files); + } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { + taskScheduler.scheduleIngestTasks(this); } + checkForStageCompleted(); } /** @@ -755,7 +751,7 @@ final class IngestJobPipeline { * pipeline after all of the files have been streamed in. See * addStreamingIngestFiles() and addStreamingIngestDataSource(). */ - private void startFirstStageInStreamingMode() { + private void startFirstStageFilesOnly() { if (doUI) { /* * If running with a GUI, start ingest progress bars in the lower @@ -775,19 +771,17 @@ final class IngestJobPipeline { } } - synchronized (stageTransitionLock) { - logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS - stage = Stages.FIRST_STAGE_STREAMING; - if (hasDataArtifactIngestModules()) { - /* - * Schedule artifact ingest tasks for any artifacts currently in - * the case database. This needs to be done before any files or - * the data source are streamed in to avoid analyzing data - * artifacts added to the case database by the data source level - * or file level ingest tasks. - */ - taskScheduler.scheduleDataArtifactIngestTasks(this); - } + logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS + stage = Stages.FIRST_STAGE_FILES_ONLY; + if (hasDataArtifactIngestModules()) { + /* + * Schedule artifact ingest tasks for any artifacts currently in the + * case database. This needs to be done before any files or the data + * source are streamed in to avoid analyzing data artifacts added to + * the case database by the data source level or file level ingest + * tasks. + */ + taskScheduler.scheduleDataArtifactIngestTasks(this); } } @@ -818,24 +812,21 @@ final class IngestJobPipeline { } } - synchronized (stageTransitionLock) { - logInfoMessage("Adding the data source in streaming mode"); //NON-NLS - stage = IngestJobPipeline.Stages.FIRST_STAGE; - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + logInfoMessage("Adding the data source in streaming mode"); //NON-NLS + stage = IngestJobPipeline.Stages.FIRST_STAGE; + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - if (hasFirstStageDataSourceIngestModules()) { - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - /* - * If no data source level ingest task is scheduled at this time - * and all of the file level and artifact ingest tasks scheduled - * when streaming began have already executed, there will never - * be a stage completion check in an ingest thread executing an - * ingest task, so such a job would run forever without the - * check here. - */ - checkForStageCompleted(); - } + if (hasFirstStageDataSourceIngestModules()) { + IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + /* + * If no data source level ingest task is scheduled at this time and + * all of the file level and artifact ingest tasks scheduled when + * streaming began have already executed, there will never be a + * stage completion check in an ingest thread executing an ingest + * task, so such a job would run forever without the check here. + */ + checkForStageCompleted(); } } @@ -843,10 +834,10 @@ final class IngestJobPipeline { * Starts the second stage ingest task pipelines. */ private void startSecondStage() { - if (doUI) { - startDataSourceIngestProgressBar(); - } - synchronized (stageTransitionLock) { + if (!cancelled && hasSecondStageDataSourceIngestModules()) { + if (doUI) { + startDataSourceIngestProgressBar(); + } logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), job.getId())); //NON-NLS stage = IngestJobPipeline.Stages.SECOND_STAGE; currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; @@ -940,19 +931,17 @@ final class IngestJobPipeline { * completed and does a stage transition if they are. */ private void checkForStageCompleted() { - synchronized (stageTransitionLock) { - if (stage == Stages.FIRST_STAGE_STREAMING) { - return; - } - if (taskScheduler.currentTasksAreCompleted(this)) { - switch (stage) { - case FIRST_STAGE: - finishFirstStage(); - break; - case SECOND_STAGE: - shutDown(); - break; - } + if (stage == Stages.FIRST_STAGE_FILES_ONLY) { + return; + } + if (taskScheduler.currentTasksAreCompleted(this)) { + switch (stage) { + case FIRST_STAGE: + finishFirstStage(); + break; + case SECOND_STAGE: + shutDown(); + break; } } } @@ -984,40 +973,38 @@ final class IngestJobPipeline { * Shuts down the ingest pipelines and progress bars for this job. */ private void shutDown() { - synchronized (stageTransitionLock) { - logInfoMessage("Finished all tasks"); //NON-NLS - stage = IngestJobPipeline.Stages.FINALIZATION; + logInfoMessage("Finished all tasks"); //NON-NLS + stage = IngestJobPipeline.Stages.FINALIZATION; - shutDownIngestTaskPipeline(currentDataSourceIngestPipeline); - shutDownIngestTaskPipeline(artifactIngestPipeline); + shutDownIngestTaskPipeline(currentDataSourceIngestPipeline); + shutDownIngestTaskPipeline(artifactIngestPipeline); - finishProgressBar(dataSourceIngestProgressBar, dataSourceIngestProgressLock); - finishProgressBar(fileIngestProgressBar, fileIngestProgressLock); - finishProgressBar(artifactIngestProgressBar, artifactIngestProgressLock); + finishProgressBar(dataSourceIngestProgressBar, dataSourceIngestProgressLock); + finishProgressBar(fileIngestProgressBar, fileIngestProgressLock); + finishProgressBar(artifactIngestProgressBar, artifactIngestProgressLock); - if (ingestJobInfo != null) { - if (cancelled) { - try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } - } else { - try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } - } + if (ingestJobInfo != null) { + if (cancelled) { try { - ingestJobInfo.setEndDateTime(new Date()); + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } + } else { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); } } - - job.notifyIngestPipelineShutDown(this); + try { + ingestJobInfo.setEndDateTime(new Date()); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); + } } + + job.notifyIngestPipelineShutDown(this); } /** @@ -1180,13 +1167,11 @@ final class IngestJobPipeline { * @param fileObjIds The object IDs of the files. */ void addStreamingIngestFiles(List fileObjIds) { - synchronized (stageTransitionLock) { - if (hasFileIngestModules()) { - if (stage.equals(Stages.FIRST_STAGE_STREAMING)) { - IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); - } + if (hasFileIngestModules()) { + if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY)) { + IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } } } @@ -1199,23 +1184,21 @@ final class IngestJobPipeline { * @param files A list of the files to add. */ void addFiles(List files) { - synchronized (stageTransitionLock) { - if (stage.equals(Stages.FIRST_STAGE_STREAMING) - || stage.equals(Stages.FIRST_STAGE)) { - taskScheduler.fastTrackFileIngestTasks(this, files); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); - } - - /** - * The intended clients of this method are ingest modules running - * code in an ingest thread that is holding a reference to a - * "primary" ingest task that was the source of the files, in which - * case a completion check would not be necessary, so this is a bit - * of defensive programming. - */ - checkForStageCompleted(); + if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY) + || stage.equals(Stages.FIRST_STAGE)) { + taskScheduler.fastTrackFileIngestTasks(this, files); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } + + /** + * The intended clients of this method are ingest modules running code + * in an ingest thread that is holding a reference to a "primary" ingest + * task that was the source of the files, in which case a completion + * check would not be necessary, so this is a bit of defensive + * programming. + */ + checkForStageCompleted(); } /** @@ -1225,24 +1208,23 @@ final class IngestJobPipeline { * @param artifacts */ void addDataArtifacts(List artifacts) { - synchronized (stageTransitionLock) { - if (stage.equals(Stages.FIRST_STAGE_STREAMING) - || stage.equals(Stages.FIRST_STAGE) - || stage.equals(Stages.SECOND_STAGE)) { - taskScheduler.scheduleDataArtifactIngestTasks(this, artifacts); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); - } - - /** - * The intended clients of this method are ingest modules running - * code in an ingest thread that is holding a reference to a - * "primary" ingest task that was the source of the files, in which - * case a completion check would not be necessary, so this is a bit - * of defensive programming. - */ - checkForStageCompleted(); + List artifactsToAnalyze = new ArrayList<>(artifacts); + if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY) + || stage.equals(Stages.FIRST_STAGE) + || stage.equals(Stages.SECOND_STAGE)) { + taskScheduler.scheduleDataArtifactIngestTasks(this, artifactsToAnalyze); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } + + /** + * The intended clients of this method are ingest modules running code + * in an ingest thread that is holding a reference to a "primary" ingest + * task that was the source of the files, in which case a completion + * check would not be necessary, so this is a bit of defensive + * programming. + */ + checkForStageCompleted(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index f14f9518c1..c09856fbc6 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -189,7 +189,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { resultIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-results-ingest-%d").build()); //NON-NLS; threadId = nextIngestManagerTaskId.incrementAndGet(); - dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getResultIngestTaskQueue())); + resultIngestTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getResultIngestTaskQueue())); // RJCTODO // ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); // RJCTODO: Where is the shut down code? diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 5568b0f3e4..eb3f7e614f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * + * * Copyright 2014-2018 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -165,6 +165,8 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { private class IngestJobTableModel extends AbstractTableModel { + private static final long serialVersionUID = 1L; + private final String[] columnNames = {NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.jobID"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dataSource"), @@ -184,7 +186,10 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.streamingQueued"), NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.dsQueued")}; + "IngestJobTableModel.colName.dsQueued"), + NbBundle.getMessage(this.getClass(), + "IngestJobTableModel.colName.artifactsQueued")}; + private List jobSnapshots; private IngestJobTableModel() { @@ -250,6 +255,9 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { case 10: cellValue = snapShot.getDsQueueSize(); break; + case 11: + cellValue = snapShot.getArtifactTasksQueueSize(); + break; default: cellValue = null; break; @@ -260,6 +268,8 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { private class ModuleTableModel extends AbstractTableModel { + private static final long serialVersionUID = 1L; + private class ModuleStats implements Comparable { private final String name; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 530ff1845b..b059cfb96b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -66,7 +66,7 @@ final class IngestTasksScheduler { @GuardedBy("this") private final Queue streamedFileIngestTasksQueue; private final IngestTaskTrackingQueue fileIngestTasksQueue; - private final IngestTaskTrackingQueue resultIngestTasksQueue; + private final IngestTaskTrackingQueue artifactIngestTasksQueue; /** * Gets the ingest tasks scheduler singleton that creates ingest tasks for @@ -91,7 +91,7 @@ final class IngestTasksScheduler { batchedFileIngestTasksQueue = new LinkedList<>(); fileIngestTasksQueue = new IngestTaskTrackingQueue(); streamedFileIngestTasksQueue = new LinkedList<>(); - resultIngestTasksQueue = new IngestTaskTrackingQueue(); + artifactIngestTasksQueue = new IngestTaskTrackingQueue(); } /** @@ -121,7 +121,7 @@ final class IngestTasksScheduler { * @return The queue. */ BlockingIngestTaskQueue getResultIngestTaskQueue() { - return resultIngestTasksQueue; + return artifactIngestTasksQueue; } /** @@ -323,7 +323,7 @@ final class IngestTasksScheduler { for (DataArtifact artifact : artifacts) { DataArtifactIngestTask task = new DataArtifactIngestTask(ingestPipeline, artifact); try { - this.resultIngestTasksQueue.putLast(task); + this.artifactIngestTasksQueue.putLast(task); } catch (InterruptedException ex) { DataSource dataSource = ingestPipeline.getDataSource(); logger.log(Level.WARNING, String.format("Interrupted while enqueuing data artifact tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS @@ -362,7 +362,7 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(DataArtifactIngestTask task) { - resultIngestTasksQueue.taskCompleted(task); + artifactIngestTasksQueue.taskCompleted(task); } /** @@ -380,7 +380,7 @@ final class IngestTasksScheduler { || hasTasksForJob(batchedFileIngestTasksQueue, pipelineId) || hasTasksForJob(streamedFileIngestTasksQueue, pipelineId) || fileIngestTasksQueue.hasTasksForJob(pipelineId) - || resultIngestTasksQueue.hasTasksForJob(pipelineId)); + || artifactIngestTasksQueue.hasTasksForJob(pipelineId)); } /** @@ -758,8 +758,9 @@ final class IngestTasksScheduler { countTasksForJob(topLevelFileIngestTasksQueue, jobId), countTasksForJob(batchedFileIngestTasksQueue, jobId), fileIngestTasksQueue.countQueuedTasksForJob(jobId), - dataSourceIngestTasksQueue.countRunningTasksForJob(jobId) + fileIngestTasksQueue.countRunningTasksForJob(jobId), - countTasksForJob(streamedFileIngestTasksQueue, jobId)); + dataSourceIngestTasksQueue.countRunningTasksForJob(jobId) + fileIngestTasksQueue.countRunningTasksForJob(jobId) + artifactIngestTasksQueue.countRunningTasksForJob(jobId), + countTasksForJob(streamedFileIngestTasksQueue, jobId), + artifactIngestTasksQueue.countQueuedTasksForJob(jobId)); } /** @@ -1063,14 +1064,24 @@ final class IngestTasksScheduler { private final long fileQueueSize; private final long runningListSize; private final long streamingQueueSize; + private final long artifactsQueueSize; /** + * RJCTODO + * * Constructs a snapshot of ingest tasks data for an ingest job. * - * @param jobId The identifier associated with the job. + * @param jobId The identifier associated with the job. + * @param dsQueueSize + * @param rootQueueSize + * @param dirQueueSize + * @param fileQueueSize + * @param runningListSize + * @param streamingQueueSize + * @param artifactsQueueSize */ IngestJobTasksSnapshot(long jobId, long dsQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, - long runningListSize, long streamingQueueSize) { + long runningListSize, long streamingQueueSize, long artifactsQueueSize) { this.jobId = jobId; this.dsQueueSize = dsQueueSize; this.rootQueueSize = rootQueueSize; @@ -1078,6 +1089,7 @@ final class IngestTasksScheduler { this.fileQueueSize = fileQueueSize; this.runningListSize = runningListSize; this.streamingQueueSize = streamingQueueSize; + this.artifactsQueueSize = artifactsQueueSize; } /** @@ -1125,6 +1137,10 @@ final class IngestTasksScheduler { long getRunningListSize() { return runningListSize; } + + long getArtifactsQueueSize() { + return artifactsQueueSize; + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java index dc7eceaad8..9a66992900 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java @@ -193,6 +193,13 @@ public final class Snapshot implements Serializable { return this.tasksSnapshot.getRunningListSize(); } + long getArtifactTasksQueueSize() { + if (tasksSnapshot == null) { + return 0; + } + return tasksSnapshot.getArtifactsQueueSize(); + } + boolean isCancelled() { return this.jobCancelled; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java index c63a7b9a43..7811635900 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java @@ -154,28 +154,23 @@ public class EXIFProcessor implements PictureProcessor { final Blackboard blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard(); if (!attributes.isEmpty() && !blackboard.artifactExists(file, TSK_METADATA_EXIF, attributes)) { - - final BlackboardArtifact exifArtifact = file.newDataArtifact(BlackboardArtifact.Type.TSK_METADATA_EXIF, attributes); - - final BlackboardArtifact userSuspectedArtifact = file.newAnalysisResult( + List artifacts = new ArrayList<>(); + final DataArtifact exifArtifact = file.newDataArtifact(BlackboardArtifact.Type.TSK_METADATA_EXIF, attributes); + artifacts.add(exifArtifact); + final AnalysisResult userSuspectedArtifact = file.newAnalysisResult( BlackboardArtifact.Type.TSK_USER_CONTENT_SUSPECTED, Score.SCORE_UNKNOWN, null, null, null, Arrays.asList(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, Bundle.ExifProcessor_userContent_description()))) .getAnalysisResult(); + artifacts.add(userSuspectedArtifact); try { - // index the artifact for keyword search - blackboard.postArtifact(exifArtifact, MODULE_NAME); - blackboard.postArtifact(userSuspectedArtifact, MODULE_NAME); + blackboard.postArtifacts(artifacts, MODULE_NAME); } catch (Blackboard.BlackboardException ex) { logger.log(Level.SEVERE, String.format("Error posting TSK_METADATA_EXIF and TSK_USER_CONTENT_SUSPECTED artifacts for %s (object ID = %d)", file.getName(), file.getId()), ex); //NON-NLS MessageNotifyUtil.Notify.error( Bundle.ExifProcessor_indexError_message(), exifArtifact.getDisplayName()); } - /* - * Add the data artifact to the ingest job for processing by the - * data artifact ingest modules. - */ context.addDataArtifactsToJob(Collections.singletonList(exifArtifact)); } } catch (TskCoreException ex) { From b89c87062954992bca7dfeb8a2d48baa1e478997 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 3 Jun 2021 07:18:21 -0400 Subject: [PATCH 16/63] 7332 artifact pipeline work --- .../autopsy/ingest/IngestJobPipeline.java | 118 ++++++++---------- 1 file changed, 52 insertions(+), 66 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 0af30ab5d9..d8084ceb9d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -55,13 +55,9 @@ import org.sleuthkit.datamodel.DataArtifact; import org.sleuthkit.datamodel.DataSource; /** - * An ingest-job-level pipeline that works with the ingest tasks scheduler to - * coordinate the creation, scheduling, and execution of ingest tasks for one of - * the data sources in an ingest job. An ingest job pipeline is actually - * composed of multiple ingest task pipelines. Each ingest task pipeline is a - * sequence of ingest modules of a given type (e.g., data source level, file - * level, or artifact ingest modules) that have been enabled and configured as - * part of the ingest job settings. + * A pipeline of ingest modules for analyzing one of the data sources in an + * ingest job. The ingest modules are actually organized into child pipelines by + * ingest module type and are run in stages. */ final class IngestJobPipeline { @@ -79,15 +75,15 @@ final class IngestJobPipeline { /* * These fields define an ingest pipeline: the parent ingest job, a pipeline * ID, the user's ingest job settings, and the data source to be analyzed. - * Optionally, there is a set of files to be analyzed, instead of analyzing + * Optionally, there is a set of files to be analyzed instead of analyzing * all of the files in the data source. * - * The pipeline ID is used to associate the pipeline with its ingest tasks. - * The ingest job ID cannot be used for this purpose because the parent - * ingest job may have more than one data source and each data source gets - * its own pipeline. + * The pipeline ID is used to associate the pipeline with the ingest tasks + * that the ingest task scheduler creates for the ingest job. The ingest job + * ID cannot be used for this purpose because the parent ingest job may have + * more than one data source and each data source gets its own pipeline. */ - private final IngestJob job; + private final IngestJob parentJob; private static final AtomicLong nextPipelineId = new AtomicLong(0L); private final long pipelineId; private final IngestJobSettings settings; @@ -110,9 +106,9 @@ final class IngestJobPipeline { */ FIRST_STAGE_FILES_ONLY, /* - * The pipeline is running one or more of the following three types of - * ingest modules: higher priority data source level ingest modules, - * file ingest modules, and artifact ingest modules. + * The pipeline is running the following three types of ingest modules: + * higher priority data source level ingest modules, file ingest + * modules, and artifact ingest modules. */ FIRST_STAGE, /** @@ -129,7 +125,7 @@ final class IngestJobPipeline { private final Object stageTransitionLock = new Object(); /* - * An ingest pipeline has separate data source level ingest task pipelines + * An ingest pipeline has separate data source level ingest module pipelines * for the first and second stages. Longer running, lower priority modules * belong in the second stage pipeline. */ @@ -138,30 +134,32 @@ final class IngestJobPipeline { private volatile DataSourceIngestPipeline currentDataSourceIngestPipeline; /* - * An ingest pipeline has a collection of identical file ingest task + * An ingest pipeline has a collection of identical file ingest module * pipelines, one for each file ingest thread in the ingest manager. The - * ingest threads take ingest task pipelines as they need them and return - * the pipelines using a blocking queue. Additionally, a fixed list of all - * of the file pipelines is used to cycle through each of the individual - * task pipelines to check their status. + * file ingest threads take file ingest pipeline copies as they need them + * for each file ingest task in the ingest job and return the pipelines + * using a blocking queue. Additionally, a fixed list of all of the file + * ingest module pipelines is used to bypass the blocking queue when cycling + * through the pipelines to make ingest progress snapshots. */ private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); private final List fileIngestPipelines = new ArrayList<>(); /* - * An ingest pipeline has a single artifact ingest task pipeline + * An ingest pipeline has a single data artifact ingest module pipeline. */ private DataArtifactIngestPipeline artifactIngestPipeline; /* - * An ingest pipeline supports cancellation of just its currently running - * data source level ingest task pipeline or cancellation of all of its - * child ingest task pipelines. Cancellation works by setting flags that are - * checked by the ingest task pipelines every time they transition from one - * module to another. Modules are also expected to check these flags (via - * the ingest job context) and stop processing if they are set. This means - * that there can be a variable length delay between a cancellation request - * and its fulfillment. + * An ingest pipeline supports cancellation of analysis by individual data + * source level ingest modules or cancellation of all remaining analysis by + * ingest modules. Cancellation works by setting flags that are checked by + * the ingest module pipelines every time they transition from one module to + * another. Ingest modules are also expected to check these flags (via the + * ingest job context) and stop processing if they are set. This approach to + * cancellation means that there can be a variable length delay between a + * cancellation request and its fulfillment. Analysis already completed at + * the time that cancellation occurs is not discarded. */ private volatile boolean currentDataSourceIngestModuleCancelled; private final List cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>(); @@ -169,19 +167,17 @@ final class IngestJobPipeline { private volatile IngestJob.CancellationReason cancellationReason = IngestJob.CancellationReason.NOT_CANCELLED; /* - * An ingest pipeline interacts with the ingest task scheduler to create and - * queue ingest tasks and to determine whether or not there are ingest tasks - * still to be executed. + * An ingest pipeline interacts with the ingest task scheduler to create + * ingest tasks for the content of the data source that is the subject of + * the ingest job and to queue them for the ingest manager's ingest threads. + * Ingest tasks are the units of work for the child ingest module pipelines. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); /* - * If running in a GUI, an ingest pipeline reports progress and allows a - * user to cancel either an individual data source level ingest module or - * all of its ingest tasks using progress bars in the lower right hand - * corner of the main application window. There is also support for taking - * ingest progress snapshots and for recording ingest job details in the - * case database. + * If running with a GUI, an ingest pipeline reports analysis progress and + * allows a user to cancel all or part of the analysis using progress bars + * in the lower right hand corner of the main application window. */ private final boolean doUI; private final Object dataSourceIngestProgressLock = new Object(); @@ -195,8 +191,8 @@ final class IngestJobPipeline { private ProgressHandle artifactIngestProgressBar; /* - * Ingest pipeline details are tracked using this object and are recorded in - * the case database. + * Ingest job details are tracked using this object and are recorded in the + * case database. */ private volatile IngestJobInfo ingestJobInfo; @@ -206,18 +202,13 @@ final class IngestJobPipeline { private final long createTime; /** - * Constructs an ingest-job-level pipeline that works with the ingest tasks - * scheduler to coordinate the creation, scheduling, and execution of ingest - * tasks for one of the data sources in an ingest job. An ingest job - * pipeline is actually composed of multiple ingest task pipelines. Each - * ingest task pipeline is a sequence of ingest modules of a given type - * (e.g., data source level, file level, or artifact ingest modules) that - * have been enabled and configured as part of the ingest job settings. + * Constructs a pipeline of ingest modules for analyzing one of the data + * sources in an ingest job. The ingest modules are actually organized into + * child pipelines by ingest module type and are run in stages. * * @param job The ingest job. - * @param dataSource One of the data sources that are the subjects of the - * ingest job. - * @param settings The ingest settings for the ingest job. + * @param dataSource The data source. + * @param settings The ingest job settings. * * @throws InterruptedException Exception thrown if the thread in which the * pipeline is being created is interrupted. @@ -227,21 +218,16 @@ final class IngestJobPipeline { } /** - * Constructs an ingest-job-level pipeline that works with the ingest tasks - * scheduler to coordinate the creation, scheduling, and execution of ingest - * tasks for one of the data sources in an ingest job. An ingest job - * pipeline is actually composed of multiple ingest task pipelines. Each - * ingest task pipeline is a sequence of ingest modules of a given type - * (e.g., data source level, file level, or artifact ingest modules) that - * have been enabled and configured as part of the ingest job settings. + * Constructs a pipeline of ingest modules for analyzing one of the data + * sources in an ingest job. The ingest modules are actually organized into + * child pipelines by ingest module type and are run in stages. * * @param job The ingest job. - * @param dataSource One of the data sources that are the subjects of the - * ingest job. + * @param dataSource The data source. * @param files A subset of the files from the data source. If the list * is empty, ALL of the files in the data source are an * analyzed. - * @param settings The ingest settings for the ingest job. + * @param settings The ingest job settings. * * @throws InterruptedException Exception thrown if the thread in which the * pipeline is being created is interrupted. @@ -250,7 +236,7 @@ final class IngestJobPipeline { if (!(dataSource instanceof DataSource)) { throw new IllegalArgumentException("Passed dataSource that does not implement the DataSource interface"); //NON-NLS } - this.job = job; + parentJob = job; pipelineId = IngestJobPipeline.nextPipelineId.getAndIncrement(); this.dataSource = (DataSource) dataSource; this.files = new ArrayList<>(); @@ -577,7 +563,7 @@ final class IngestJobPipeline { if (errors.isEmpty()) { recordIngestJobStartUpInfo(); if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { - if (job.getIngestMode() == IngestJob.Mode.STREAMING) { + if (parentJob.getIngestMode() == IngestJob.Mode.STREAMING) { startFirstStageFilesOnly(); } else { startFirstStage(); @@ -838,7 +824,7 @@ final class IngestJobPipeline { if (doUI) { startDataSourceIngestProgressBar(); } - logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), job.getId())); //NON-NLS + logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS stage = IngestJobPipeline.Stages.SECOND_STAGE; currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; taskScheduler.scheduleDataSourceIngestTask(this); @@ -1004,7 +990,7 @@ final class IngestJobPipeline { } } - job.notifyIngestPipelineShutDown(this); + parentJob.notifyIngestPipelineShutDown(this); } /** From 85ab8d7c3d7b0ab1f7938e55d80325eecdfbc114 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 3 Jun 2021 14:57:49 -0400 Subject: [PATCH 17/63] 7332 artifact pipeline work --- .../sleuthkit/autopsy/ingest/IngestJob.java | 4 +- .../autopsy/ingest/IngestJobPipeline.java | 248 ++++++++++-------- 2 files changed, 135 insertions(+), 117 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index a82c3878b0..9793a57cf1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -161,7 +161,7 @@ public final class IngestJob { } // Streaming ingest jobs will only have one data source IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); - streamingIngestPipeline.addStreamingIngestFiles(fileObjIds); + streamingIngestPipeline.addStreamedFiles(fileObjIds); } /** @@ -174,7 +174,7 @@ public final class IngestJob { } // Streaming ingest jobs will only have one data source IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); - streamingIngestPipeline.notifyFileStreamingCompleted(); + streamingIngestPipeline.notifyDataSourceReady(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index d8084ceb9d..c0715c6658 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -56,8 +56,8 @@ import org.sleuthkit.datamodel.DataSource; /** * A pipeline of ingest modules for analyzing one of the data sources in an - * ingest job. The ingest modules are actually organized into child pipelines by - * ingest module type and are run in stages. + * ingest job. The ingest modules are organized into child pipelines by ingest + * module type and are run in stages. */ final class IngestJobPipeline { @@ -101,10 +101,11 @@ final class IngestJobPipeline { INITIALIZATION, /* * The pipeline is running file ingest modules on files streamed to it - * by a data source processor. The data source has not been added to the - * pipeline yet. + * by a data source processor and data artifact ingest modules on + * artifacts generated by the analysis of the streamed files. The data + * source has not been added to the pipeline yet. */ - FIRST_STAGE_FILES_ONLY, + FILE_STREAMING, /* * The pipeline is running the following three types of ingest modules: * higher priority data source level ingest modules, file ingest @@ -203,26 +204,26 @@ final class IngestJobPipeline { /** * Constructs a pipeline of ingest modules for analyzing one of the data - * sources in an ingest job. The ingest modules are actually organized into - * child pipelines by ingest module type and are run in stages. + * sources in an ingest job. The ingest modules are organized into child + * pipelines by ingest module type and are run in stages. * - * @param job The ingest job. + * @param parentJob The ingest job. * @param dataSource The data source. * @param settings The ingest job settings. * * @throws InterruptedException Exception thrown if the thread in which the * pipeline is being created is interrupted. */ - IngestJobPipeline(IngestJob job, Content dataSource, IngestJobSettings settings) throws InterruptedException { - this(job, dataSource, Collections.emptyList(), settings); + IngestJobPipeline(IngestJob parentJob, Content dataSource, IngestJobSettings settings) throws InterruptedException { + this(parentJob, dataSource, Collections.emptyList(), settings); } /** * Constructs a pipeline of ingest modules for analyzing one of the data - * sources in an ingest job. The ingest modules are actually organized into - * child pipelines by ingest module type and are run in stages. + * sources in an ingest job. The ingest modules are organized into child + * pipelines by ingest module type and are run in stages. * - * @param job The ingest job. + * @param parentJob The ingest job. * @param dataSource The data source. * @param files A subset of the files from the data source. If the list * is empty, ALL of the files in the data source are an @@ -232,11 +233,11 @@ final class IngestJobPipeline { * @throws InterruptedException Exception thrown if the thread in which the * pipeline is being created is interrupted. */ - IngestJobPipeline(IngestJob job, Content dataSource, List files, IngestJobSettings settings) throws InterruptedException { + IngestJobPipeline(IngestJob parentJob, Content dataSource, List files, IngestJobSettings settings) throws InterruptedException { if (!(dataSource instanceof DataSource)) { throw new IllegalArgumentException("Passed dataSource that does not implement the DataSource interface"); //NON-NLS } - parentJob = job; + this.parentJob = parentJob; pipelineId = IngestJobPipeline.nextPipelineId.getAndIncrement(); this.dataSource = (DataSource) dataSource; this.files = new ArrayList<>(); @@ -245,22 +246,21 @@ final class IngestJobPipeline { doUI = RuntimeProperties.runningWithGUI(); createTime = new Date().getTime(); stage = Stages.INITIALIZATION; - createIngestTaskPipelines(); + createIngestModulePipelines(); } /** - * Adds ingest module templates to an output list with core Autopsy modules - * first and third party modules next. + * Sorts ingest module templates so that core Autopsy ingest modules come + * before third party ingest modules and ingest modules implemented using + * Java come before ingest modules implemented using Jython. * - * @param orderedModules The list to populate. - * @param javaModules A map of the input ingest module templates for - * modules implemented using Java, keyed by - * fully-qualified (canonical) class name. - * @param jythonModules A map of the input ingest module templates for - * modules implemented using Jython, keyed by - * fully-qualified (canonical) class name. + * @param sortedModules The output list to hold the sorted modules. + * @param javaModules The input ingest module templates for modules + * implemented using Java. + * @param jythonModules The ingest module templates for modules implemented + * using Jython. */ - private static void completePipeline(final List orderedModules, final Map javaModules, final Map jythonModules) { + private static void sortModuleTemplates(final List sortedModules, final Map javaModules, final Map jythonModules) { final List autopsyModules = new ArrayList<>(); final List thirdPartyModules = new ArrayList<>(); Stream.concat(javaModules.entrySet().stream(), jythonModules.entrySet().stream()).forEach((templateEntry) -> { @@ -270,8 +270,8 @@ final class IngestJobPipeline { thirdPartyModules.add(templateEntry.getValue()); } }); - orderedModules.addAll(autopsyModules); - orderedModules.addAll(thirdPartyModules); + sortedModules.addAll(autopsyModules); + sortedModules.addAll(thirdPartyModules); } /** @@ -304,7 +304,7 @@ final class IngestJobPipeline { * @param jythonMapping Mapping for Jython ingest module templates. * @param template The ingest module template. */ - private static void addIngestModuleTemplateToMaps(Map mapping, Map jythonMapping, IngestModuleTemplate template) { + private static void addModuleTemplateToImplLangMap(Map mapping, Map jythonMapping, IngestModuleTemplate template) { String className = template.getModuleFactory().getClass().getCanonicalName(); String jythonName = getModuleNameFromJythonClassName(className); if (jythonName != null) { @@ -315,13 +315,12 @@ final class IngestJobPipeline { } /** - * Creates the child ingest task pipelines for this ingest pipeline. + * Creates the child ingest module pipelines for this ingest pipeline. * * @throws InterruptedException Exception thrown if the thread in which the - * task pipelines are being created is - * interrupted. + * pipeline is being created is interrupted. */ - private void createIngestTaskPipelines() throws InterruptedException { + private void createIngestModulePipelines() throws InterruptedException { /* * Get the enabled ingest module templates from the ingest job settings. * An ingest module template combines an ingest module factory with job @@ -352,13 +351,13 @@ final class IngestJobPipeline { Map jythonArtifactModuleTemplates = new LinkedHashMap<>(); for (IngestModuleTemplate template : enabledTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - addIngestModuleTemplateToMaps(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); + addModuleTemplateToImplLangMap(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); } if (template.isFileIngestModuleTemplate()) { - addIngestModuleTemplateToMaps(javaFileModuleTemplates, jythonFileModuleTemplates, template); + addModuleTemplateToImplLangMap(javaFileModuleTemplates, jythonFileModuleTemplates, template); } if (template.isDataArtifactIngestModuleTemplate()) { - addIngestModuleTemplateToMaps(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); + addModuleTemplateToImplLangMap(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); } } @@ -370,9 +369,9 @@ final class IngestJobPipeline { * source level and file ingest module pipeline layouts. */ IngestPipelinesConfiguration pipelineConfig = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = createPipelineFromConfigFile(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageOneDataSourceIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = createPipelineFromConfigFile(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageTwoDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = createPipelineFromConfigFile(javaFileModuleTemplates, jythonFileModuleTemplates, pipelineConfig.getFileIngestPipelineConfig()); + List firstStageDataSourceModuleTemplates = addConfiguredIngestModuleTemplates(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageOneDataSourceIngestPipelineConfig()); + List secondStageDataSourceModuleTemplates = addConfiguredIngestModuleTemplates(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageTwoDataSourceIngestPipelineConfig()); + List fileIngestModuleTemplates = addConfiguredIngestModuleTemplates(javaFileModuleTemplates, jythonFileModuleTemplates, pipelineConfig.getFileIngestPipelineConfig()); List artifactModuleTemplates = new ArrayList<>(); /** @@ -383,9 +382,9 @@ final class IngestJobPipeline { * modules, and Core Autopsy modules are added before third party * modules. */ - completePipeline(firstStageDataSourceModuleTemplates, javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates); - completePipeline(fileIngestModuleTemplates, javaFileModuleTemplates, jythonFileModuleTemplates); - completePipeline(artifactModuleTemplates, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); + sortModuleTemplates(firstStageDataSourceModuleTemplates, javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates); + sortModuleTemplates(fileIngestModuleTemplates, javaFileModuleTemplates, jythonFileModuleTemplates); + sortModuleTemplates(artifactModuleTemplates, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); /** * Construct the actual ingest task pipelines from the ordered lists. @@ -422,7 +421,7 @@ final class IngestJobPipeline { * @return An ordered list of ingest module templates, i.e., an * uninstantiated pipeline. */ - private static List createPipelineFromConfigFile(Map javaIngestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { + private static List addConfiguredIngestModuleTemplates(Map javaIngestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { List templates = new ArrayList<>(); for (String moduleClassName : pipelineConfig) { if (javaIngestModuleTemplates.containsKey(moduleClassName)) { @@ -559,12 +558,12 @@ final class IngestJobPipeline { * @return A collection of ingest module startup errors, empty on success. */ List startUp() { - List errors = startUpIngestTaskPipelines(); + List errors = startUpIngestModulePipelines(); if (errors.isEmpty()) { recordIngestJobStartUpInfo(); if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { if (parentJob.getIngestMode() == IngestJob.Mode.STREAMING) { - startFirstStageFilesOnly(); + startFileStreaming(); } else { startFirstStage(); } @@ -626,7 +625,7 @@ final class IngestJobPipeline { } /** - * Starts up each of the child ingest task pipelines in this ingest + * Starts up each of the child ingest module pipelines in this ingest * pipeline. * * Note that all of the child pipelines are started so that any and all @@ -638,12 +637,12 @@ final class IngestJobPipeline { * * @return A list of ingest module startup errors, empty on success. */ - private List startUpIngestTaskPipelines() { + private List startUpIngestModulePipelines() { List errors = new ArrayList<>(); - errors.addAll(startUpIngestTaskPipeline(firstStageDataSourceIngestPipeline)); - errors.addAll(startUpIngestTaskPipeline(secondStageDataSourceIngestPipeline)); + errors.addAll(startUpIngestModulePipeline(firstStageDataSourceIngestPipeline)); + errors.addAll(startUpIngestModulePipeline(secondStageDataSourceIngestPipeline)); for (FileIngestPipeline pipeline : fileIngestPipelines) { - List filePipelineErrors = startUpIngestTaskPipeline(pipeline); + List filePipelineErrors = startUpIngestModulePipeline(pipeline); if (!filePipelineErrors.isEmpty()) { /* * If one file pipeline copy can't start up, assume that none of @@ -653,19 +652,19 @@ final class IngestJobPipeline { break; } } - errors.addAll(startUpIngestTaskPipeline(artifactIngestPipeline)); + errors.addAll(startUpIngestModulePipeline(artifactIngestPipeline)); return errors; } /** - * Starts up an ingest task pipeline. If there are any start up errors, the - * pipeline is immediately shut down. + * Starts up an ingest module pipeline. If there are any start up errors, + * the pipeline is immediately shut down. * * @param pipeline The ingest task pipeline to start up. * * @return A list of ingest module startup errors, empty on success. */ - private List startUpIngestTaskPipeline(IngestTaskPipeline pipeline) { + private List startUpIngestModulePipeline(IngestTaskPipeline pipeline) { List startUpErrors = pipeline.startUp(); if (!startUpErrors.isEmpty()) { List shutDownErrors = pipeline.shutDown(); @@ -682,6 +681,9 @@ final class IngestJobPipeline { * already been added to the case database by the data source processor. */ private void startFirstStage() { + logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS + stage = Stages.FIRST_STAGE; + /* * Do a count of the files the data source processor has added to the * case database. This estimate will be used for ingest progress @@ -710,17 +712,19 @@ final class IngestJobPipeline { } } - logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS - stage = Stages.FIRST_STAGE; + /* + * Make the first stage data source level ingest pipeline the current + * data source level pipeline. + */ currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - /** + /* * Schedule the first stage ingest tasks and then immediately check for * stage completion. This is necessary because it is possible that zero * tasks will actually make it to task execution due to the file filter * or other ingest job settings. In that case, there will never be a * stage completion check in an ingest thread executing an ingest task, - * so such a job would run forever without the check here. + * so such a job would run forever without a check here. */ if (!files.isEmpty() && hasFileIngestModules()) { taskScheduler.scheduleFileIngestTasks(this, files); @@ -733,11 +737,13 @@ final class IngestJobPipeline { /** * Starts the first stage of this pipeline in streaming mode. In streaming * mode, the data source processor streams files into the pipeline as it - * adds them to the case database and only adds the data source to the - * pipeline after all of the files have been streamed in. See - * addStreamingIngestFiles() and addStreamingIngestDataSource(). + * adds them to the case database and file level analysis can begin before + * data source level analysis. */ - private void startFirstStageFilesOnly() { + private void startFileStreaming() { + logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS + stage = Stages.FILE_STREAMING; + if (doUI) { /* * If running with a GUI, start ingest progress bars in the lower @@ -746,9 +752,9 @@ final class IngestJobPipeline { if (hasFileIngestModules()) { /* * Note that because estimated files remaining to process still - * has its initial value of zero since files are still be added - * to the case database, the progress bar will start in the - * "indeterminate" state. + * has its initial value of zero, the progress bar will start in + * the "indeterminate" state. An estimate of the files to + * process can be computed in */ startFileIngestProgressBar(); } @@ -757,8 +763,6 @@ final class IngestJobPipeline { } } - logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS - stage = Stages.FIRST_STAGE_FILES_ONLY; if (hasDataArtifactIngestModules()) { /* * Schedule artifact ingest tasks for any artifacts currently in the @@ -772,10 +776,14 @@ final class IngestJobPipeline { } /** - * Start data source ingest. Used for streaming ingest when the data source - * is not ready when ingest starts. + * Notifies the ingest pipeline running in streaming mode that the data + * source is now ready for analysis. */ - void notifyFileStreamingCompleted() { + void notifyDataSourceReady() { + logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS + stage = IngestJobPipeline.Stages.FIRST_STAGE; + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + /* * Do a count of the files the data source processor has added to the * case database. This estimate will be used for ingest progress @@ -788,20 +796,13 @@ final class IngestJobPipeline { estimatedFilesToProcess = filesToProcess; } - /* - * If running with a GUI, start ingest progress bars in the lower right - * hand corner of the main application window. - */ if (doUI) { if (hasFirstStageDataSourceIngestModules()) { startDataSourceIngestProgressBar(); } } - logInfoMessage("Adding the data source in streaming mode"); //NON-NLS - stage = IngestJobPipeline.Stages.FIRST_STAGE; currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - if (hasFirstStageDataSourceIngestModules()) { IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); } else { @@ -810,7 +811,7 @@ final class IngestJobPipeline { * all of the file level and artifact ingest tasks scheduled when * streaming began have already executed, there will never be a * stage completion check in an ingest thread executing an ingest - * task, so such a job would run forever without the check here. + * task, so such a job would run forever without a check here. */ checkForStageCompleted(); } @@ -821,11 +822,13 @@ final class IngestJobPipeline { */ private void startSecondStage() { if (!cancelled && hasSecondStageDataSourceIngestModules()) { + logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS + stage = IngestJobPipeline.Stages.SECOND_STAGE; + if (doUI) { startDataSourceIngestProgressBar(); } - logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS - stage = IngestJobPipeline.Stages.SECOND_STAGE; + currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; taskScheduler.scheduleDataSourceIngestTask(this); } @@ -917,7 +920,7 @@ final class IngestJobPipeline { * completed and does a stage transition if they are. */ private void checkForStageCompleted() { - if (stage == Stages.FIRST_STAGE_FILES_ONLY) { + if (stage == Stages.FILE_STREAMING) { return; } if (taskScheduler.currentTasksAreCompleted(this)) { @@ -939,14 +942,27 @@ final class IngestJobPipeline { private void finishFirstStage() { logInfoMessage("Finished first stage analysis"); //NON-NLS - shutDownIngestTaskPipeline(currentDataSourceIngestPipeline); + shutDownIngestModulePipeline(currentDataSourceIngestPipeline); while (!fileIngestPipelinesQueue.isEmpty()) { FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); - shutDownIngestTaskPipeline(pipeline); + shutDownIngestModulePipeline(pipeline); } - finishProgressBar(dataSourceIngestProgressBar, dataSourceIngestProgressLock); - finishProgressBar(fileIngestProgressBar, fileIngestProgressLock); + if (doUI) { + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + } + + synchronized (fileIngestProgressLock) { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + } + } if (!cancelled && hasSecondStageDataSourceIngestModules()) { startSecondStage(); @@ -956,19 +972,38 @@ final class IngestJobPipeline { } /** - * Shuts down the ingest pipelines and progress bars for this job. + * Shuts down the ingest module pipelines and progress bars for this job. */ private void shutDown() { logInfoMessage("Finished all tasks"); //NON-NLS stage = IngestJobPipeline.Stages.FINALIZATION; - shutDownIngestTaskPipeline(currentDataSourceIngestPipeline); - shutDownIngestTaskPipeline(artifactIngestPipeline); + shutDownIngestModulePipeline(currentDataSourceIngestPipeline); + shutDownIngestModulePipeline(artifactIngestPipeline); - finishProgressBar(dataSourceIngestProgressBar, dataSourceIngestProgressLock); - finishProgressBar(fileIngestProgressBar, fileIngestProgressLock); - finishProgressBar(artifactIngestProgressBar, artifactIngestProgressLock); + if (doUI) { + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + } + synchronized (fileIngestProgressLock) { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + } + + synchronized (artifactIngestProgressLock) { + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.finish(); + artifactIngestProgressBar = null; + } + } + } + if (ingestJobInfo != null) { if (cancelled) { try { @@ -998,7 +1033,7 @@ final class IngestJobPipeline { * * @param pipeline The pipeline. */ - private void shutDownIngestTaskPipeline(IngestTaskPipeline pipeline) { + private void shutDownIngestModulePipeline(IngestTaskPipeline pipeline) { if (pipeline.isRunning()) { List errors = new ArrayList<>(); errors.addAll(pipeline.shutDown()); @@ -1008,23 +1043,6 @@ final class IngestJobPipeline { } } - /** - * Finishes a progress bar. - * - * @param progress The progress bar. - * @param lock The lock that guards the progress bar. - */ - private void finishProgressBar(ProgressHandle progress, Object lock) { - if (doUI) { - synchronized (lock) { - if (progress != null) { - progress.finish(); - progress = null; - } - } - } - } - /** * Passes the data source for the ingest job through the currently active * data source level ingest task pipeline (first stage or second stage data @@ -1147,14 +1165,14 @@ final class IngestJobPipeline { } /** - * Adds some subset of the "streamed" files for a streaming ingest job to - * this pipeline after startUp() has been called. + * Adds some subset of the streamed files for a streaming mode ingest job to + * this pipeline. * * @param fileObjIds The object IDs of the files. */ - void addStreamingIngestFiles(List fileObjIds) { + void addStreamedFiles(List fileObjIds) { if (hasFileIngestModules()) { - if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY)) { + if (stage.equals(Stages.FILE_STREAMING)) { IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); @@ -1170,7 +1188,7 @@ final class IngestJobPipeline { * @param files A list of the files to add. */ void addFiles(List files) { - if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY) + if (stage.equals(Stages.FILE_STREAMING) || stage.equals(Stages.FIRST_STAGE)) { taskScheduler.fastTrackFileIngestTasks(this, files); } else { @@ -1195,7 +1213,7 @@ final class IngestJobPipeline { */ void addDataArtifacts(List artifacts) { List artifactsToAnalyze = new ArrayList<>(artifacts); - if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY) + if (stage.equals(Stages.FILE_STREAMING) || stage.equals(Stages.FIRST_STAGE) || stage.equals(Stages.SECOND_STAGE)) { taskScheduler.scheduleDataArtifactIngestTasks(this, artifactsToAnalyze); From a186defacbe08929484ed75ce10872757298cf89 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 4 Jun 2021 07:03:14 -0400 Subject: [PATCH 18/63] 7332 artifact pipeline work --- .../autopsy/actions/Bundle.properties-MERGED | 8 + .../casemodule/Bundle.properties-MERGED | 27 +- .../Bundle.properties-MERGED | 5 +- .../contentviewers/Bundle.properties-MERGED | 1337 +++++++++-------- .../autopsy/core/Bundle.properties-MERGED | 8 +- .../corecomponents/Bundle.properties-MERGED | 6 +- .../coreutils/Bundle.properties-MERGED | 4 +- .../datamodel/Bundle.properties-MERGED | 9 +- .../filesearch/Bundle.properties-MERGED | 4 +- .../autopsy/ingest/Bundle.properties-MERGED | 2 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 2 +- .../autopsy/ingest/IngestJobPipeline.java | 103 +- .../Bundle.properties-MERGED | 7 +- .../fileextmismatch/Bundle.properties-MERGED | 18 +- .../hashdatabase/Bundle.properties-MERGED | 10 +- .../interestingitems/Bundle.properties-MERGED | 5 +- .../photoreccarver/Bundle.properties-MERGED | 2 +- .../modules/html/Bundle.properties-MERGED | 6 +- 18 files changed, 834 insertions(+), 729 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties-MERGED index cc5c9222e4..5c9a0ea3ac 100755 --- a/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/actions/Bundle.properties-MERGED @@ -1,18 +1,24 @@ AddBlackboardArtifactTagAction.pluralTagResult=Add Result Tags AddBlackboardArtifactTagAction.singularTagResult=Add Result Tag AddBlackboardArtifactTagAction.taggingErr=Tagging Error +# {0} - artifactName AddBlackboardArtifactTagAction.unableToTag.msg=Unable to tag {0}. AddContentTagAction.cannotApplyTagErr=Cannot Apply Tag AddContentTagAction.pluralTagFile=Add File Tags AddContentTagAction.singularTagFile=Add File Tag +# {0} - fileName +# {1} - tagName AddContentTagAction.tagExists={0} has been tagged as {1}. Cannot reapply the same tag. AddContentTagAction.taggingErr=Tagging Error +# {0} - fileName AddContentTagAction.unableToTag.msg=Unable to tag {0}, not a regular file. +# {0} - fileName AddContentTagAction.unableToTag.msg2=Unable to tag {0}. CTL_DumpThreadAction=Thread Dump CTL_ShowIngestProgressSnapshotAction=Ingest Status Details DeleteBlackboardArtifactTagAction.deleteTag=Remove Selected Tag(s) DeleteBlackboardArtifactTagAction.tagDelErr=Tag Deletion Error +# {0} - tagName DeleteBlackboardArtifactTagAction.unableToDelTag.msg=Unable to delete tag {0}. DeleteContentTagAction.deleteTag=Remove Selected Tag(s) DeleteContentTagAction.tagDelErr=Tag Deletion Error @@ -78,6 +84,8 @@ CTL_OpenOutputFolder=Open Case Folder OpenOutputFolder.error1=Case Folder Not Found: {0} OpenOutputFolder.noCaseOpen=No open case, therefore no current case folder available. OpenOutputFolder.CouldNotOpenOutputFolder=Could not open case folder +# {0} - old tag name +# {1} - artifactID ReplaceBlackboardArtifactTagAction.replaceTag.alert=Unable to replace tag {0} for artifact {1}. # {0} - old tag name # {1} - content obj id diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED index 9a43ffe229..528d3a5088 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED @@ -247,10 +247,15 @@ AddImageWizardIngestConfigPanel.dsProcDone.errs.text=*Errors encountered in addi AddImageWizardIngestConfigVisual.getName.text=Configure Ingest AddImageWizardIterator.stepXofN=Step {0} of {1} AddLocalFilesTask.localFileAdd.progress.text=Adding: {0}/{1} -Case.getCurCase.exception.noneOpen=Cannot get the current case; there is no case open! +Case.getCurCase.exception.noneOpen=Cannot get the current case; there is no case open\! Case.open.msgDlg.updated.msg=Updated case database schema.\nA backup copy of the database with the following path has been made:\n {0} Case.open.msgDlg.updated.title=Case Database Schema Update -Case.checkImgExist.confDlg.doesntExist.msg=One of the images associated with \nthis case are missing. Would you like to search for them now?\nPreviously, the image was located at:\n{0}\nPlease note that you will still be able to browse directories and generate reports\nif you choose No, but you will not be able to view file content or run the ingest process. +Case.checkImgExist.confDlg.doesntExist.msg=One of the images associated with \n\ +this case are missing. Would you like to search for them now?\n\ +Previously, the image was located at:\n\ +{0}\n\ +Please note that you will still be able to browse directories and generate reports\n\ +if you choose No, but you will not be able to view file content or run the ingest process. Case.checkImgExist.confDlg.doesntExist.title=Missing Image Case.addImg.exception.msg=Error adding image to the case Case.updateCaseName.exception.msg=Error while trying to update the case name. @@ -269,9 +274,12 @@ Case.GetCaseTypeGivenPath.Failure=Unable to get case type Case.metaDataFileCorrupt.exception.msg=The case metadata file (.aut) is corrupted. Case.deleteReports.deleteFromDiskException.log.msg=Unable to delete the report from the disk. Case.deleteReports.deleteFromDiskException.msg=Unable to delete the report {0} from the disk.\nYou may manually delete it from {1} -CaseDeleteAction.closeConfMsg.text=Are you sure want to close and delete this case? \nCase Name: {0}\nCase Directory: {1} +CaseDeleteAction.closeConfMsg.text=Are you sure want to close and delete this case? \n\ + Case Name: {0}\n\ + Case Directory: {1} CaseDeleteAction.closeConfMsg.title=Warning: Closing the Current Case -CaseDeleteAction.msgDlg.fileInUse.msg=The delete action cannot be fully completed because the folder or file in it is open by another program.\n\nClose the folder and file and try again or you can delete the case manually. +CaseDeleteAction.msgDlg.fileInUse.msg=The delete action cannot be fully completed because the folder or file in it is open by another program.\n\n\ +Close the folder and file and try again or you can delete the case manually. CaseDeleteAction.msgDlg.fileInUse.title=Error: Folder In Use CaseDeleteAction.msgDlg.caseDelete.msg=Case {0} has been deleted. CaseOpenAction.autFilter.title={0} Case File ( {1}) @@ -303,7 +311,8 @@ NewCaseWizardAction.databaseProblem1.text=Cannot open database. Cancelling case NewCaseWizardAction.databaseProblem2.text=Error NewCaseWizardPanel1.validate.errMsg.invalidSymbols=The Case Name cannot contain any of the following symbols: \\ / : * ? " < > | NewCaseWizardPanel1.validate.errMsg.dirExists=Case directory ''{0}'' already exists. -NewCaseWizardPanel1.validate.confMsg.createDir.msg=The base directory "{0}" does not exist. \n\nDo you want to create that directory? +NewCaseWizardPanel1.validate.confMsg.createDir.msg=The base directory "{0}" does not exist. \n\n\ + Do you want to create that directory? NewCaseWizardPanel1.validate.confMsg.createDir.title=Create directory NewCaseWizardPanel1.validate.errMsg.cantCreateParDir.msg=Error: Could not create case parent directory {0} NewCaseWizardPanel1.validate.errMsg.prevCreateBaseDir.msg=Prevented from creating base directory {0} @@ -332,6 +341,7 @@ OptionalCasePropertiesPanel.lbPointOfContactPhoneLabel.text=Phone: OptionalCasePropertiesPanel.orgainizationPanel.border.title=Organization RecentCases.exception.caseIdxOutOfRange.msg=Recent case index {0} is out of range. RecentCases.getName.text=Clear Recent Cases +# {0} - case name RecentItems.openRecentCase.msgDlg.text=Case {0} no longer exists. SelectDataSourceProcessorPanel.name.text=Select Data Source Type StartupWindow.title.text=Welcome @@ -344,6 +354,7 @@ StartupWindowProvider.openCase.noFile=Unable to open previously open case becaus UnpackagePortableCaseDialog.title.text=Unpackage Portable Case UnpackagePortableCaseDialog.UnpackagePortableCaseDialog.extensions=Portable case package (.zip, .zip.001) UnpackagePortableCaseDialog.validatePaths.badExtension=File extension must be .zip or .zip.001 +# {0} - case folder UnpackagePortableCaseDialog.validatePaths.caseFolderExists=Folder {0} already exists UnpackagePortableCaseDialog.validatePaths.caseIsNotFile=Selected path is not a file UnpackagePortableCaseDialog.validatePaths.caseNotFound=File does not exist @@ -358,8 +369,8 @@ UnpackageWorker.doInBackground.previouslySeenCase=Case has been previously opene UpdateRecentCases.menuItem.clearRecentCases.text=Clear Recent Cases UpdateRecentCases.menuItem.empty=-Empty- AddImageWizardIngestConfigPanel.CANCEL_BUTTON.text=Cancel -NewCaseVisualPanel1.CaseFolderOnCDriveError.text=Warning: Path to multi-user case folder is on "C:" drive -NewCaseVisualPanel1.CaseFolderOnInternalDriveWindowsError.text=Warning: Path to case folder is on "C:" drive. Case folder is created on the target system +NewCaseVisualPanel1.CaseFolderOnCDriveError.text=Warning: Path to multi-user case folder is on \"C:\" drive +NewCaseVisualPanel1.CaseFolderOnInternalDriveWindowsError.text=Warning: Path to case folder is on \"C:\" drive. Case folder is created on the target system NewCaseVisualPanel1.CaseFolderOnInternalDriveLinuxError.text=Warning: Path to case folder is on the target system. Create case folder in mounted drive. NewCaseVisualPanel1.uncPath.error=Error: UNC paths are not allowed for Single-User cases CollaborationMonitor.addingDataSourceStatus.msg={0} adding data source @@ -367,7 +378,7 @@ CollaborationMonitor.analyzingDataSourceStatus.msg={0} analyzing {1} MissingImageDialog.lbWarning.text= MissingImageDialog.lbWarning.toolTipText= NewCaseVisualPanel1.caseParentDirWarningLabel.text= -NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-User\t\t +NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-User NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-User NewCaseVisualPanel1.caseTypeLabel.text=Case Type: SingleUserCaseConverter.BadDatabaseFileName=Database file does not exist! diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/centralrepository/Bundle.properties-MERGED index b2320b5408..b4f7f835ef 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/Bundle.properties-MERGED @@ -5,7 +5,10 @@ CentralRepoCommentDialog.title.addEditCentralRepoComment=Add/Edit Central Reposi OpenIDE-Module-Name=Central Repository OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Short-Description=Central Repository Ingest Module -OpenIDE-Module-Long-Description=Central Repository ingest module and central database. \n\nThe Central Repository ingest module stores attributes of artifacts matching selected correlation types into a central database.\nStored attributes are used in future cases to correlate and analyzes files and artifacts during ingest. +OpenIDE-Module-Long-Description=\ + Central Repository ingest module and central database. \n\n\ + The Central Repository ingest module stores attributes of artifacts matching selected correlation types into a central database.\n\ + Stored attributes are used in future cases to correlate and analyzes files and artifacts during ingest. CentralRepoCommentDialog.commentLabel.text=Comment: CentralRepoCommentDialog.okButton.text=&OK CentralRepoCommentDialog.cancelButton.text=C&ancel diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED index 53aaacb575..dacd037aaa 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED @@ -181,776 +181,837 @@ TranslatablePanel.comboBoxOption.translatedText=Translated Text # {0} - exception message TranslatablePanel.onSetContentError.text=There was an error displaying the text: {0} ## Window toolbar Title -viewer.window.title.default=ICEpdf Viewer -viewer.window.title.open.default=ICEpdf Viewer - [{0}] +viewer.window.title.default = ICEpdf Viewer +viewer.window.title.open.default = ICEpdf Viewer - [{0}] #status bar -viewer.statusbar.currentPage=Page {0} / {1} +viewer.statusbar.currentPage = Page {0} / {1} -viewer.common.number.one=1 -viewer.common.number.two=2 -viewer.common.number.three=3 -viewer.common.number.four=4 -viewer.common.number.five=5 -viewer.common.number.six=6 -viewer.common.number.seven=7 -viewer.common.number.eight=8 -viewer.common.number.nine=9 -viewer.common.number.ten=10 -viewer.common.number.eleven=11 -viewer.common.number.twelve=12 -viewer.common.number.thirteen=13 -viewer.common.number.fourteen=14 -viewer.common.number.fifteen=15 -viewer.common.number.sixteen=16 -viewer.common.number.seventeen=17 -viewer.common.number.eighteen=18 -viewer.common.number.nineteen=19 -viewer.common.number.twenty=20 -viewer.common.number.twentyOne=21 -viewer.common.number.twentyTwo=22 -viewer.common.number.twentyThree=23 -viewer.common.number.twentyFour=24 -viewer.common.number.twentyFive=25 -viewer.common.number.twentySix=26 -viewer.common.number.twentySeven=27 -viewer.common.number.thirtySix=36 -viewer.common.number.fortyEight=48 +viewer.common.number.one = 1 +viewer.common.number.two = 2 +viewer.common.number.three = 3 +viewer.common.number.four = 4 +viewer.common.number.five = 5 +viewer.common.number.six = 6 +viewer.common.number.seven = 7 +viewer.common.number.eight = 8 +viewer.common.number.nine = 9 +viewer.common.number.ten = 10 +viewer.common.number.eleven = 11 +viewer.common.number.twelve = 12 +viewer.common.number.thirteen = 13 +viewer.common.number.fourteen = 14 +viewer.common.number.fifteen = 15 +viewer.common.number.sixteen = 16 +viewer.common.number.seventeen = 17 +viewer.common.number.eighteen = 18 +viewer.common.number.nineteen = 19 +viewer.common.number.twenty = 20 +viewer.common.number.twentyOne = 21 +viewer.common.number.twentyTwo = 22 +viewer.common.number.twentyThree = 23 +viewer.common.number.twentyFour = 24 +viewer.common.number.twentyFive = 25 +viewer.common.number.twentySix = 26 +viewer.common.number.twentySeven = 27 +viewer.common.number.thirtySix = 36 +viewer.common.number.fortyEight = 48 ## Top Page Control Toolbar -viewer.toolbar.hideToolBar.label=Hide Toolbar -viewer.toolbar.showToolBar.label=Show Toolbar -viewer.toolbar.showUtilityPane.label=Show Utility Pane -viewer.toolbar.hideUtilityPane.label=Hide Utility Pane -viewer.toolbar.open.label= -viewer.toolbar.open.tooltip=Open Document -viewer.toolbar.saveAs.label=Save As -viewer.toolbar.saveAs.tooltip=Save As... -viewer.toolbar.print.label=Print -viewer.toolbar.print.tooltip=Print Document -viewer.toolbar.search.label=Search -viewer.toolbar.search.tooltip=Search Document -viewer.toolbar.utilityPane.label=Utility Pane -viewer.toolbar.utilityPane.tooltip=Show/Hide Utility Pane -viewer.toolbar.navigation.label= -viewer.toolbar.navigation.pages.tooltip=Number of Pages -viewer.toolbar.navigation.pages.firstPage.label= -viewer.toolbar.navigation.current.tooltip=Current Page Number -viewer.toolbar.navigation.current.firstPage.label= -viewer.toolbar.navigation.firstPage.label= -viewer.toolbar.navigation.firstPage.tooltip=First Page -viewer.toolbar.navigation.previousPage.label= -viewer.toolbar.navigation.previousPage.tooltip=Previous Page -viewer.toolbar.navigation.nextPage.label= -viewer.toolbar.navigation.nextPage.tooltip=Next Page -viewer.toolbar.navigation.lastPage.label= -viewer.toolbar.navigation.lastPage.tooltip=Last Page -viewer.toolbar.pageIndicator=of {0} -viewer.toolbar.zoom.label= -viewer.toolbar.zoom.tooltip=Zoom -viewer.toolbar.zoom.out.label= -viewer.toolbar.zoom.out.tooltip=Zoom Out -viewer.toolbar.zoom.in.label= -viewer.toolbar.zoom.in.tooltip=Zoom In -viewer.toolbar.pageFit.actualsize.label= -viewer.toolbar.pageFit.actualsize.tooltip=Actual Size -viewer.toolbar.pageFit.fitWindow.label= -viewer.toolbar.pageFit.fitWindow.tooltip=Fit in Window -viewer.toolbar.pageFit.fitWidth.label= -viewer.toolbar.pageFit.fitWidth.tooltip=Fit Width -viewer.toolbar.rotation.left.label= -viewer.toolbar.rotation.left.tooltip=Rotate Left -viewer.toolbar.rotation.right.label= -viewer.toolbar.rotation.right.tooltip=Rotate Right -viewer.toolbar.tool.pan.label= -viewer.toolbar.tool.pan.tooltip=Pan Tool -viewer.toolbar.tool.text.label= -viewer.toolbar.tool.text.tooltip=Text Select Tool -viewer.toolbar.tool.select.label= -viewer.toolbar.tool.select.tooltip=Select Tool -viewer.toolbar.tool.link.label= -viewer.toolbar.tool.link.tooltip=Link Annotation Tool -viewer.toolbar.tool.highlight.label=Highlight -viewer.toolbar.tool.highlight.tooltip=Highlight Annotation Tool -viewer.toolbar.tool.strikeOut.label=Strike Out -viewer.toolbar.tool.strikeOut.tooltip=Strike Out Annotation Tool -viewer.toolbar.tool.underline.label=Underline -viewer.toolbar.tool.underline.tooltip=Underline Annotation Tool -viewer.toolbar.tool.line.label=Line -viewer.toolbar.tool.line.tooltip=Line Annotation Tool -viewer.toolbar.tool.lineArrow.label=Line Arrow -viewer.toolbar.tool.lineArrow.tooltip=Line Arrow Annotation Tool -viewer.toolbar.tool.rectangle.label=Rectangle -viewer.toolbar.tool.rectangle.tooltip=Rectangle Annotation Tool -viewer.toolbar.tool.circle.label=Circle -viewer.toolbar.tool.circle.tooltip=Circle Annotation Tool -viewer.toolbar.tool.ink.label=Ink -viewer.toolbar.tool.ink.tooltip=Ink Annotation Tool -viewer.toolbar.tool.freeText.label=Free Text -viewer.toolbar.tool.freeText.tooltip=Free Text Annotation Tool -viewer.toolbar.tool.textAnno.label=Text Annotation -viewer.toolbar.tool.textAnno.tooltip=Text Annotation Tool -viewer.toolbar.tool.plolyLine.label=Poly Line -viewer.toolbar.tool.plolyLine.tooltip=Poly Line Annotation Tool -viewer.toolbar.tool.zoomIn.label= -viewer.toolbar.tool.zoomIn.tooltip=Zoom In Tool -viewer.toolbar.tool.zoomMarquis.label= -viewer.toolbar.tool.zoomMarquis.tooltip=Zoom Marquee Tool -viewer.toolbar.tool.zoomDynamic.label= -viewer.toolbar.tool.zoomDynamic.tooltip=Zoom Dynamic Tool -viewer.toolbar.tool.zoomOut.label= -viewer.toolbar.tool.zoomOut.tooltip=Zoom Out Tool -viewer.toolbar.pageFit.fontEngine.label= -viewer.toolbar.pageFit.fontEngine.tooltip=Enable/Disable Font Engine -viewer.toolbar.tool.forms.highlight.label=Highlight Forms -viewer.toolbar.tool.forms.highlight.tooltip=Show/Hide Form Highlighting +viewer.toolbar.hideToolBar.label = Hide Toolbar +viewer.toolbar.showToolBar.label = Show Toolbar +viewer.toolbar.showUtilityPane.label = Show Utility Pane +viewer.toolbar.hideUtilityPane.label = Hide Utility Pane +viewer.toolbar.open.label = +viewer.toolbar.open.tooltip = Open Document +viewer.toolbar.saveAs.label = Save As +viewer.toolbar.saveAs.tooltip = Save As... +viewer.toolbar.print.label = Print +viewer.toolbar.print.tooltip = Print Document +viewer.toolbar.search.label = Search +viewer.toolbar.search.tooltip = Search Document +viewer.toolbar.utilityPane.label = Utility Pane +viewer.toolbar.utilityPane.tooltip = Show/Hide Utility Pane +viewer.toolbar.navigation.label = +viewer.toolbar.navigation.pages.tooltip = Number of Pages +viewer.toolbar.navigation.pages.firstPage.label = +viewer.toolbar.navigation.current.tooltip = Current Page Number +viewer.toolbar.navigation.current.firstPage.label = +viewer.toolbar.navigation.firstPage.label = +viewer.toolbar.navigation.firstPage.tooltip = First Page +viewer.toolbar.navigation.previousPage.label = +viewer.toolbar.navigation.previousPage.tooltip = Previous Page +viewer.toolbar.navigation.nextPage.label = +viewer.toolbar.navigation.nextPage.tooltip = Next Page +viewer.toolbar.navigation.lastPage.label = +viewer.toolbar.navigation.lastPage.tooltip = Last Page +viewer.toolbar.pageIndicator = of {0} +viewer.toolbar.zoom.label = +viewer.toolbar.zoom.tooltip = Zoom +viewer.toolbar.zoom.out.label = +viewer.toolbar.zoom.out.tooltip = Zoom Out +viewer.toolbar.zoom.in.label = +viewer.toolbar.zoom.in.tooltip = Zoom In +viewer.toolbar.pageFit.actualsize.label = +viewer.toolbar.pageFit.actualsize.tooltip = Actual Size +viewer.toolbar.pageFit.fitWindow.label = +viewer.toolbar.pageFit.fitWindow.tooltip = Fit in Window +viewer.toolbar.pageFit.fitWidth.label = +viewer.toolbar.pageFit.fitWidth.tooltip = Fit Width +viewer.toolbar.rotation.left.label = +viewer.toolbar.rotation.left.tooltip = Rotate Left +viewer.toolbar.rotation.right.label = +viewer.toolbar.rotation.right.tooltip = Rotate Right +viewer.toolbar.tool.pan.label = +viewer.toolbar.tool.pan.tooltip = Pan Tool +viewer.toolbar.tool.text.label = +viewer.toolbar.tool.text.tooltip = Text Select Tool +viewer.toolbar.tool.select.label = +viewer.toolbar.tool.select.tooltip = Select Tool +viewer.toolbar.tool.link.label = +viewer.toolbar.tool.link.tooltip = Link Annotation Tool +viewer.toolbar.tool.highlight.label = Highlight +viewer.toolbar.tool.highlight.tooltip = Highlight Annotation Tool +viewer.toolbar.tool.strikeOut.label = Strike Out +viewer.toolbar.tool.strikeOut.tooltip = Strike Out Annotation Tool +viewer.toolbar.tool.underline.label = Underline +viewer.toolbar.tool.underline.tooltip = Underline Annotation Tool +viewer.toolbar.tool.line.label = Line +viewer.toolbar.tool.line.tooltip = Line Annotation Tool +viewer.toolbar.tool.lineArrow.label = Line Arrow +viewer.toolbar.tool.lineArrow.tooltip = Line Arrow Annotation Tool +viewer.toolbar.tool.rectangle.label = Rectangle +viewer.toolbar.tool.rectangle.tooltip = Rectangle Annotation Tool +viewer.toolbar.tool.circle.label = Circle +viewer.toolbar.tool.circle.tooltip = Circle Annotation Tool +viewer.toolbar.tool.ink.label = Ink +viewer.toolbar.tool.ink.tooltip = Ink Annotation Tool +viewer.toolbar.tool.freeText.label = Free Text +viewer.toolbar.tool.freeText.tooltip = Free Text Annotation Tool +viewer.toolbar.tool.textAnno.label = Text Annotation +viewer.toolbar.tool.textAnno.tooltip = Text Annotation Tool +viewer.toolbar.tool.plolyLine.label = Poly Line +viewer.toolbar.tool.plolyLine.tooltip = Poly Line Annotation Tool +viewer.toolbar.tool.zoomIn.label = +viewer.toolbar.tool.zoomIn.tooltip = Zoom In Tool +viewer.toolbar.tool.zoomMarquis.label = +viewer.toolbar.tool.zoomMarquis.tooltip = Zoom Marquee Tool +viewer.toolbar.tool.zoomDynamic.label = +viewer.toolbar.tool.zoomDynamic.tooltip = Zoom Dynamic Tool +viewer.toolbar.tool.zoomOut.label = +viewer.toolbar.tool.zoomOut.tooltip = Zoom Out Tool +viewer.toolbar.pageFit.fontEngine.label = +viewer.toolbar.pageFit.fontEngine.tooltip = Enable/Disable Font Engine +viewer.toolbar.tool.forms.highlight.label = Highlight Forms +viewer.toolbar.tool.forms.highlight.tooltip = Show/Hide Form Highlighting ## Bottom Page View Control Toolbar -viewer.toolbar.pageView.nonContinuous.singlePage.label= -viewer.toolbar.pageView.nonContinuous.singlePage.tooltip=Single Page View Non-Continuous -viewer.toolbar.pageView.nonContinuous.facingPage.label= -viewer.toolbar.pageView.nonContinuous.facingPage.tooltip=Facing Page View Non-Continuous -viewer.toolbar.pageView.continuous.singlePage.label= -viewer.toolbar.pageView.continuous.singlePage.tooltip=Single Page View Continuous -viewer.toolbar.pageView.continuous.facingPage.label= -viewer.toolbar.pageView.continuous.facingPage.tooltip=Facing Page View Continuous +viewer.toolbar.pageView.nonContinuous.singlePage.label = +viewer.toolbar.pageView.nonContinuous.singlePage.tooltip = Single Page View Non-Continuous +viewer.toolbar.pageView.nonContinuous.facingPage.label = +viewer.toolbar.pageView.nonContinuous.facingPage.tooltip = Facing Page View Non-Continuous +viewer.toolbar.pageView.continuous.singlePage.label = +viewer.toolbar.pageView.continuous.singlePage.tooltip = Single Page View Continuous +viewer.toolbar.pageView.continuous.facingPage.label = +viewer.toolbar.pageView.continuous.facingPage.tooltip = Facing Page View Continuous ## File Menu and submenu items -viewer.menu.file.label=File -viewer.menu.file.mnemonic=F -viewer.menu.open.label=Open -viewer.menu.open.file.label=File... -viewer.menu.open.URL.label=URL... -viewer.menu.close.label=Close -viewer.menu.saveAs.label=Save As... -viewer.menu.exportText.label=Export Text... -viewer.menu.exportSVG.label=Export SVG... -viewer.menu.documentPermission.label=Document Permissions... -viewer.menu.documentInformation.label=Document Information... -viewer.menu.documentFonts.label=Document Fonts... -viewer.menu.printSetup.label=Print Setup... -viewer.menu.print.label=Print... -viewer.menu.exit.label=Exit +viewer.menu.file.label = File +viewer.menu.file.mnemonic = F +viewer.menu.open.label = Open +viewer.menu.open.file.label = File... +viewer.menu.open.URL.label = URL... +viewer.menu.close.label = Close +viewer.menu.saveAs.label = Save As... +viewer.menu.exportText.label = Export Text... +viewer.menu.exportSVG.label = Export SVG... +viewer.menu.documentPermission.label = Document Permissions... +viewer.menu.documentInformation.label = Document Information... +viewer.menu.documentFonts.label = Document Fonts... +viewer.menu.printSetup.label = Print Setup... +viewer.menu.print.label = Print... +viewer.menu.exit.label = Exit ## View Menu and submenu items -viewer.menu.edit.label=Edit -viewer.menu.edit.mnemonic=E -viewer.menu.edit.undo.label=Undo -viewer.menu.edit.redo.label=Redo -viewer.menu.edit.copy.label=Copy -viewer.menu.edit.delete.label=Delete -viewer.menu.edit.selectAll.label=Select All -viewer.menu.edit.deselectAll.label=Deselect All +viewer.menu.edit.label = Edit +viewer.menu.edit.mnemonic = E +viewer.menu.edit.undo.label = Undo +viewer.menu.edit.redo.label = Redo +viewer.menu.edit.copy.label = Copy +viewer.menu.edit.delete.label = Delete +viewer.menu.edit.selectAll.label = Select All +viewer.menu.edit.deselectAll.label = Deselect All ## View Menu and submenu items -viewer.menu.view.label=View -viewer.menu.view.mnemonic=V -viewer.menu.view.actualSize.label=Actual Size -viewer.menu.view.fitInWindow.label=Fit in Window -viewer.menu.view.fitWidth.label=Fit Width -viewer.menu.view.zoomIn.label=Zoom In -viewer.menu.view.zoomOut.label=Zoom Out -viewer.menu.view.rotateLeft.label=Rotate Left -viewer.menu.view.rotateRight.label=Rotate Right -viewer.menu.view.hideToolBar.label=Hide Toolbar -viewer.menu.view.showToolBar.label=Show Toolbar -viewer.menu.view.showUtilityPane.label=Show Utility Pane -viewer.menu.view.hideUtilityPane.label=Hide Utility Pane +viewer.menu.view.label = View +viewer.menu.view.mnemonic = V +viewer.menu.view.actualSize.label = Actual Size +viewer.menu.view.fitInWindow.label = Fit in Window +viewer.menu.view.fitWidth.label = Fit Width +viewer.menu.view.zoomIn.label = Zoom In +viewer.menu.view.zoomOut.label = Zoom Out +viewer.menu.view.rotateLeft.label = Rotate Left +viewer.menu.view.rotateRight.label = Rotate Right +viewer.menu.view.hideToolBar.label = Hide Toolbar +viewer.menu.view.showToolBar.label = Show Toolbar +viewer.menu.view.showUtilityPane.label = Show Utility Pane +viewer.menu.view.hideUtilityPane.label = Hide Utility Pane ## Document Menu and submenu items -viewer.menu.document.label=Document -viewer.menu.document.mnemonic=D -viewer.menu.document.firstPage.label=First Page -viewer.menu.document.previousPage.label=Previous Page -viewer.menu.document.nextPage.label=Next Page -viewer.menu.document.lastPage.label=Last Page -viewer.menu.document.search.label=Search... -viewer.menu.document.gotToPage.label=Go To Page... +viewer.menu.document.label = Document +viewer.menu.document.mnemonic = D +viewer.menu.document.firstPage.label = First Page +viewer.menu.document.previousPage.label = Previous Page +viewer.menu.document.nextPage.label = Next Page +viewer.menu.document.lastPage.label = Last Page +viewer.menu.document.search.label = Search... +viewer.menu.document.gotToPage.label = Go To Page... ## Window Menu and submenu items -viewer.menu.window.label=Window -viewer.menu.window.mnemonic=W -viewer.menu.window.minAll.label=Minimize All -viewer.menu.window.minAll.mnemonic=M -viewer.menu.window.frontAll.label=Bring All to Front -viewer.menu.window.frontAll.mnemonic=B -viewer.menu.window.1.label=1 -viewer.menu.window.1.mnemonic=1 -viewer.menu.window.2.label=2 -viewer.menu.window.2.mnemonic=2 -viewer.menu.window.3.label=3 -viewer.menu.window.3.mnemonic=3 -viewer.menu.window.4.label=4 -viewer.menu.window.4.mnemonic=4 -viewer.menu.window.5.label=5 -viewer.menu.window.5.mnemonic=5 -viewer.menu.window.6.label=6 -viewer.menu.window.6.mnemonic=6 -viewer.menu.window.7.label=7 -viewer.menu.window.7.mnemonic=7 -viewer.menu.window.8.label=8 -viewer.menu.window.8.mnemonic=8 -viewer.menu.window.9.label=9 -viewer.menu.window.9.mnemonic=9 +viewer.menu.window.label = Window +viewer.menu.window.mnemonic = W +viewer.menu.window.minAll.label = Minimize All +viewer.menu.window.minAll.mnemonic = M +viewer.menu.window.frontAll.label = Bring All to Front +viewer.menu.window.frontAll.mnemonic = B +viewer.menu.window.1.label = 1 +viewer.menu.window.1.mnemonic = 1 +viewer.menu.window.2.label = 2 +viewer.menu.window.2.mnemonic = 2 +viewer.menu.window.3.label = 3 +viewer.menu.window.3.mnemonic = 3 +viewer.menu.window.4.label = 4 +viewer.menu.window.4.mnemonic = 4 +viewer.menu.window.5.label = 5 +viewer.menu.window.5.mnemonic = 5 +viewer.menu.window.6.label = 6 +viewer.menu.window.6.mnemonic = 6 +viewer.menu.window.7.label = 7 +viewer.menu.window.7.mnemonic = 7 +viewer.menu.window.8.label = 8 +viewer.menu.window.8.mnemonic = 8 +viewer.menu.window.9.label = 9 +viewer.menu.window.9.mnemonic = 9 ## Add as many entries as you want, to viewer.menu.window.X.label and mnemonic ## where X is an incrementing integer. The mnemonic should be one unique ## character found within the label ## Help Menu and submenu items -viewer.menu.help.label=Help -viewer.menu.help.mnemonic=H -viewer.menu.help.about.label=About ICEpdf viewer... +viewer.menu.help.label = Help +viewer.menu.help.mnemonic = H +viewer.menu.help.about.label = About ICEpdf viewer... ## General error dialog -viewer.dialog.error.exception.title=ICEsoft ICEpdf - Exception -viewer.dialog.error.exception.msg=There was an error executing your command do to the following exception\n{0}. +viewer.dialog.error.exception.title = ICEsoft ICEpdf - Exception +viewer.dialog.error.exception.msg = \ + There was an error executing your command do to the following exception\n\ + {0}. ## Open File Dialog -viewer.dialog.openFile.title=Open File -viewer.dialog.openFile.error.title=ICEsoft ICEpdf - Open File Error -viewer.dialog.openFile.error.msg=ICEpdf could not open the specified file at {0}\nThe file may be corrupt or not a supported file type. +viewer.dialog.openFile.title = Open File +viewer.dialog.openFile.error.title = ICEsoft ICEpdf - Open File Error +viewer.dialog.openFile.error.msg = \ + ICEpdf could not open the specified file at {0}\n\ + The file may be corrupt or not a supported file type. -viewer.dialog.openDocument.pdfException.title=ICEsoft ICEpdf - PDF Exception -viewer.dialog.openDocument.pdfException.msg=ICEpdf could not open the specified file {0} \nThe file may be corrupt or not a supported file type. +viewer.dialog.openDocument.pdfException.title = ICEsoft ICEpdf - PDF Exception +viewer.dialog.openDocument.pdfException.msg = \ + ICEpdf could not open the specified file {0} \n\ + The file may be corrupt or not a supported file type. -viewer.dialog.openDocument.pdfSecurityException.title=ICEsoft ICEpdf - PDF Security Exception -viewer.dialog.openDocument.pdfSecurityException.msg=ICEpdf could not open the encrypted file at {0}\nThis may be the result of an invalid password or a missing JCE Security Provider.\n\nPlease refer to ICEpdf Developer's Guide for more information. +viewer.dialog.openDocument.pdfSecurityException.title = ICEsoft ICEpdf - PDF Security Exception +viewer.dialog.openDocument.pdfSecurityException.msg = \ + ICEpdf could not open the encrypted file at {0}\n\ + This may be the result of an invalid password or a missing JCE Security Provider.\n\n\ + Please refer to ICEpdf Developer's Guide for more information. -viewer.dialog.openDocument.exception.title=ICEsoft ICEpdf - Exception -viewer.dialog.openDocument.exception.msg=ICEpdf could not open the specified file at {0}\nThe file may be corrupt or not a supported file type. +viewer.dialog.openDocument.exception.title = ICEsoft ICEpdf - Exception +viewer.dialog.openDocument.exception.msg = \ + ICEpdf could not open the specified file at {0}\n\ + The file may be corrupt or not a supported file type. -viewer.dialog.openURL.exception.title=ICEsoft ICEpdf - URL Exception -viewer.dialog.openURL.exception.msg=ICEpdf could not open the specified file. {0} \nat URL: {1} -viewer.dialog.openURL.downloading.msg=Downloading {0} +viewer.dialog.openURL.exception.title = ICEsoft ICEpdf - URL Exception +viewer.dialog.openURL.exception.msg = \ + ICEpdf could not open the specified file. {0} \n\ + at URL: {1} +viewer.dialog.openURL.downloading.msg = Downloading {0} ## General error dialog -viewer.dialog.information.copyAll.title=ICEsoft ICEpdf - Information -viewer.dialog.information.copyAll.msg=The document has more than {0} pages, please use\n"Export text..." to extract document text. +viewer.dialog.information.copyAll.title = ICEsoft ICEpdf - Information +viewer.dialog.information.copyAll.msg = \ + The document has more than {0} pages, please use\n\ + "Export text..." to extract document text. ## Open URL Dialog -viewer.dialog.security.title=Document Security -viewer.dialog.security.msg=This PDF is protected -viewer.dialog.security.password.label=Password: -viewer.dialog.security.okButton.label=Ok -viewer.dialog.security.okButton.mnemonic=O -viewer.dialog.security.cancelButton.label=Cancel -viewer.dialog.security.cancelButton.mnemonic=C +viewer.dialog.security.title = Document Security +viewer.dialog.security.msg = This PDF is protected +viewer.dialog.security.password.label = Password: +viewer.dialog.security.okButton.label = Ok +viewer.dialog.security.okButton.mnemonic = O +viewer.dialog.security.cancelButton.label = Cancel +viewer.dialog.security.cancelButton.mnemonic = C ## Open URL Dialog -viewer.dialog.openURL.title=Open URL +viewer.dialog.openURL.title = Open URL ### Save a Copy Dialog -viewer.dialog.saveAs.title=Save As -viewer.dialog.saveAs.extensionError.title=ICEsoft ICEpdf - Save Error -viewer.dialog.saveAs.extensionError.msg=ICEpdf could not save to {0} because it is not a supported file type. -viewer.dialog.saveAs.noExtensionError.title=ICEsoft ICEpdf - Save Error -viewer.dialog.saveAs.noExtensionError.msg=Please specify a file extension. -viewer.dialog.saveAs.noneUniqueName.title=ICEsoft ICEpdf - Save Error -viewer.dialog.saveAs.noneUniqueName.msg=The file named {0} already exists. Please specify a unique name. -viewer.dialog.saveAs.noPermission.title=ICEpdf Viewer RI - Save Error -viewer.dialog.saveAs.noPermission.msg=You do not have permission or the credentials to save this document. -viewer.dialog.saveAs.noUpdates.title=ICEpdf Viewer RI -viewer.dialog.saveAs.noUpdates.msg=Document changes will not be saved, please upgrade to ICEpdf PRO. -viewer.dialog.saveOnClose.noUpdates.title=ICEpdf Viewer RI -viewer.dialog.saveOnClose.noUpdates.msg=Do you want to save changes to {0}? +viewer.dialog.saveAs.title = Save As +viewer.dialog.saveAs.extensionError.title = ICEsoft ICEpdf - Save Error +viewer.dialog.saveAs.extensionError.msg = \ + ICEpdf could not save to {0} because it is not a supported file type. +viewer.dialog.saveAs.noExtensionError.title = ICEsoft ICEpdf - Save Error +viewer.dialog.saveAs.noExtensionError.msg = Please specify a file extension. +viewer.dialog.saveAs.noneUniqueName.title = ICEsoft ICEpdf - Save Error +viewer.dialog.saveAs.noneUniqueName.msg = \ + The file named {0} already exists. Please specify a unique name. +viewer.dialog.saveAs.noPermission.title = ICEpdf Viewer RI - Save Error +viewer.dialog.saveAs.noPermission.msg = You do not have permission or the credentials to save this document. +viewer.dialog.saveAs.noUpdates.title = ICEpdf Viewer RI +viewer.dialog.saveAs.noUpdates.msg = Document changes will not be saved, please upgrade to ICEpdf PRO. +viewer.dialog.saveOnClose.noUpdates.title = ICEpdf Viewer RI +viewer.dialog.saveOnClose.noUpdates.msg = Do you want to save changes to {0}? ## Export Text Dialog -viewer.dialog.exportText.title=Export Document Text -viewer.dialog.exportText.progress.msg=Extracting PDF Text -viewer.dialog.exportText.noExtensionError.title=ICEsoft ICEpdf - Save Error -viewer.dialog.exportText.noExtensionError.msg=Please specify a file extension. +viewer.dialog.exportText.title = Export Document Text +viewer.dialog.exportText.progress.msg = Extracting PDF Text +viewer.dialog.exportText.noExtensionError.title = ICEsoft ICEpdf - Save Error +viewer.dialog.exportText.noExtensionError.msg = Please specify a file extension. # Text extraction output file -viewer.exportText.fileStamp.msg=ICEsoft ICEpdf Viewer, (c) ICEsoft Technologies, Inc. -viewer.exportText.pageStamp.msg= +viewer.exportText.fileStamp.msg = ICEsoft ICEpdf Viewer, (c) ICEsoft Technologies, Inc. +viewer.exportText.pageStamp.msg = # Completed x out of y page(s). -viewer.exportText.fileStamp.progress.msg=Completed {0} out of {1}. -viewer.exportText.fileStamp.progress.oneFile.msg={2} page -viewer.exportText.fileStamp.progress.moreFile.msg={2} pages +viewer.exportText.fileStamp.progress.msg = \ + Completed {0} out of {1}. +viewer.exportText.fileStamp.progress.oneFile.msg = {2} page +viewer.exportText.fileStamp.progress.moreFile.msg = {2} pages ## Export SVG Dialog -viewer.dialog.exportSVG.title=Export to SVG -viewer.dialog.exportSVG.status.exporting.msg=Exporting page {0} to SVG file {1} ... -viewer.dialog.exportSVG.status.error.msg=Problem exporting page {0} to SVG file {1} : {2} -viewer.dialog.exportSVG.status.finished.msg=Finished exporting page {0} to SVG file {1} -viewer.dialog.exportSVG.noExtensionError.title=ICEsoft ICEpdf - SVG Error -viewer.dialog.exportSVG.noExtensionError.msg=Please specify a file extension. -viewer.dialog.exportSVG.exportError.title=ICEsoft ICEpdf - SVG Error -viewer.dialog.exportSVG.exportError.msg=ICEpdf could not export to {0} \nbecuase it is either not a supported file type or \nbecause the file has been corrupted. +viewer.dialog.exportSVG.title = Export to SVG +viewer.dialog.exportSVG.status.exporting.msg = Exporting page {0} to SVG file {1} ... +viewer.dialog.exportSVG.status.error.msg = \ + Problem exporting page {0} to SVG file {1} : {2} +viewer.dialog.exportSVG.status.finished.msg = \ + Finished exporting page {0} to SVG file {1} +viewer.dialog.exportSVG.noExtensionError.title = ICEsoft ICEpdf - SVG Error +viewer.dialog.exportSVG.noExtensionError.msg = Please specify a file extension. +viewer.dialog.exportSVG.exportError.title = ICEsoft ICEpdf - SVG Error +viewer.dialog.exportSVG.exportError.msg = \ + ICEpdf could not export to {0} \n\ + becuase it is either not a supported file type or \n\ + because the file has been corrupted. # Printing Progress bar -viewer.dialog.printing.status.progress.msg=Page {0} of {1} -viewer.dialog.printing.status.start.msg=Spooling Page(s) to Printer +viewer.dialog.printing.status.progress.msg = Page {0} of {1} +viewer.dialog.printing.status.start.msg = Spooling Page(s) to Printer ## Document Permissions Dialog -viewer.dialog.documentPermissions.title=Document Permissions -viewer.dialog.documentPermissions.securityMethod.label=Security Method: -viewer.dialog.documentPermissions.userPassword.label=User Password: -viewer.dialog.documentPermissions.ownerPassword.label=Owner Password: -viewer.dialog.documentPermissions.printing.label=Printing: -viewer.dialog.documentPermissions.changing.label=Changing the Document: -viewer.dialog.documentPermissions.copyExtraction.label=Content Copying or Extraction: -viewer.dialog.documentPermissions.comments.label=Aurthoring Comments and Form Fields: -viewer.dialog.documentPermissions.formFillingIn.label=Form Field Fill-in or Signing: -viewer.dialog.documentPermissions.accessibility.label=Content Accessibility Enabled: -viewer.dialog.documentPermissions.assembly.label=Document Assembly: -viewer.dialog.documentPermissions.encryptionLevel.label=Encryption Level: -viewer.dialog.documentPermissions.securityLevel={0}-bit v{1} R {2} -viewer.dialog.documentPermissions.none=None -viewer.dialog.documentPermissions.no=No -viewer.dialog.documentPermissions.yes=Yes -viewer.dialog.documentPermissions.allowed=Allowed -viewer.dialog.documentPermissions.notAllowed=Not Allowed -viewer.dialog.documentPermissions.fullyAllowed=Fully Allowed -viewer.dialog.documentPermissions.standardSecurity=Adobe Acrobat Standard Security -viewer.dialog.documentPermissions.partial=Partial (Low Quality) +viewer.dialog.documentPermissions.title = Document Permissions +viewer.dialog.documentPermissions.securityMethod.label = Security Method: +viewer.dialog.documentPermissions.userPassword.label = User Password: +viewer.dialog.documentPermissions.ownerPassword.label = Owner Password: +viewer.dialog.documentPermissions.printing.label = Printing: +viewer.dialog.documentPermissions.changing.label = Changing the Document: +viewer.dialog.documentPermissions.copyExtraction.label = Content Copying or Extraction: +viewer.dialog.documentPermissions.comments.label = Aurthoring Comments and Form Fields: +viewer.dialog.documentPermissions.formFillingIn.label = Form Field Fill-in or Signing: +viewer.dialog.documentPermissions.accessibility.label = Content Accessibility Enabled: +viewer.dialog.documentPermissions.assembly.label = Document Assembly: +viewer.dialog.documentPermissions.encryptionLevel.label = Encryption Level: +viewer.dialog.documentPermissions.securityLevel = {0}-bit v{1} R {2} +viewer.dialog.documentPermissions.none = None +viewer.dialog.documentPermissions.no = No +viewer.dialog.documentPermissions.yes = Yes +viewer.dialog.documentPermissions.allowed = Allowed +viewer.dialog.documentPermissions.notAllowed = Not Allowed +viewer.dialog.documentPermissions.fullyAllowed = Fully Allowed +viewer.dialog.documentPermissions.standardSecurity = Adobe Acrobat Standard Security +viewer.dialog.documentPermissions.partial = Partial (Low Quality) ## Document Information Dialog -viewer.dialog.documentInformation.title=Document Information -viewer.dialog.documentInformation.title.label=Title: -viewer.dialog.documentInformation.subject.label=Subject: -viewer.dialog.documentInformation.author.label=Author: -viewer.dialog.documentInformation.keywords.label=Keywords: -viewer.dialog.documentInformation.creator.label=Creator: -viewer.dialog.documentInformation.producer.label=Producer: -viewer.dialog.documentInformation.created.label=Created: -viewer.dialog.documentInformation.modified.label=Modified: -viewer.dialog.documentInformation.notAvailable=Not Available +viewer.dialog.documentInformation.title = Document Information +viewer.dialog.documentInformation.title.label = Title: +viewer.dialog.documentInformation.subject.label = Subject: +viewer.dialog.documentInformation.author.label = Author: +viewer.dialog.documentInformation.keywords.label = Keywords: +viewer.dialog.documentInformation.creator.label = Creator: +viewer.dialog.documentInformation.producer.label = Producer: +viewer.dialog.documentInformation.created.label = Created: +viewer.dialog.documentInformation.modified.label = Modified: +viewer.dialog.documentInformation.notAvailable = Not Available ## Go to Page Dialog -viewer.dialog.goToPage.title=Go to Page... -viewer.dialog.goToPage.description.label=Page Number +viewer.dialog.goToPage.title = Go to Page... +viewer.dialog.goToPage.description.label = Page Number ## About Dialog -viewer.dialog.about.title=About ICEpdf Viewer -viewer.dialog.about.pageNumber.label=\n\nCheck the ICEpdf web site for the latest news:\nhttp://www.icepdf.org/ \n\n +viewer.dialog.about.title = About ICEpdf Viewer +viewer.dialog.about.pageNumber.label = \n\ +\n\ +Check the ICEpdf web site for the latest news:\n\ +http://www.icepdf.org/ \n\n ## Font Properties Dialog -viewer.dialog.fonts.title=Document Font Properties -viewer.dialog.fonts.border.label=Fonts used by this document -viewer.dialog.fonts.info.type.label=Type: {0} -viewer.dialog.fonts.info.encoding.label=Encoding: {0} -viewer.dialog.fonts.info.substitution.type.label=Actual Type: {0} -viewer.dialog.fonts.info.substitution.path.label=Path: {0} -viewer.dialog.fonts.searching.label=Collecting font data ({0}%). -viewer.dialog.fonts.resetCache.label=Reset Cache -viewer.dialog.fonts.resetCache.tip=Reset font properties cache file and rescan system for new fonts. +viewer.dialog.fonts.title = Document Font Properties +viewer.dialog.fonts.border.label = Fonts used by this document +viewer.dialog.fonts.info.type.label = Type: {0} +viewer.dialog.fonts.info.encoding.label = Encoding: {0} +viewer.dialog.fonts.info.substitution.type.label = Actual Type: {0} +viewer.dialog.fonts.info.substitution.path.label = Path: {0} +viewer.dialog.fonts.searching.label = Collecting font data ({0}%). +viewer.dialog.fonts.resetCache.label = Reset Cache +viewer.dialog.fonts.resetCache.tip = Reset font properties cache file and rescan system for new fonts. ## Utility Pane Bookmarks Tab -viewer.utilityPane.bookmarks.tab.title=Bookmarks +viewer.utilityPane.bookmarks.tab.title = Bookmarks ## Utility Pane Bookmarks Tab -viewer.utilityPane.attachments.tab.title=Attachments -viewer.utilityPane.attachments.column.fileName.title=Name -viewer.utilityPane.attachments.column.description.title=Description -viewer.utilityPane.attachments.column.modified.title=Modified -viewer.utilityPane.attachments.column.size.title=Size -viewer.utilityPane.attachments.column.compressedSize.title=Compressed size -viewer.utilityPane.attachments.menu.saveAs.label=Save As... -viewer.utilityPane.attachments.saveAs.replace.title=ICEsoft ICEpdf - Save Error -viewer.utilityPane.attachments.saveAs.replace.msg=The file named {0} already exists. Do you want to replace It? +viewer.utilityPane.attachments.tab.title = Attachments +viewer.utilityPane.attachments.column.fileName.title = Name +viewer.utilityPane.attachments.column.description.title = Description +viewer.utilityPane.attachments.column.modified.title = Modified +viewer.utilityPane.attachments.column.size.title = Size +viewer.utilityPane.attachments.column.compressedSize.title = Compressed size +viewer.utilityPane.attachments.menu.saveAs.label = Save As... +viewer.utilityPane.attachments.saveAs.replace.title = ICEsoft ICEpdf - Save Error +viewer.utilityPane.attachments.saveAs.replace.msg = \ + The file named {0} already exists. Do you want to replace It? ## Utility Pane Thumbnails -viewer.utilityPane.thumbs.tab.title=Thumbnails +viewer.utilityPane.thumbs.tab.title = Thumbnails ## Layers Pane -viewer.utilityPane.layers.tab.title=Layers +viewer.utilityPane.layers.tab.title = Layers ## Signature Pane -viewer.utilityPane.signatures.tab.title=Signatures -viewer.utilityPane.signatures.tab.certTree.error.label=Unsigned Signature Fields Signer certificate could not be validated {0} {1} -viewer.utilityPane.signatures.tab.certTree.rootSigned.label=Signed by {0} {1} -viewer.utilityPane.signatures.tab.certTree.rootValidating.label=Validating signature {0} {1} -viewer.utilityPane.signatures.tab.certTree.cert.invalid.label=Signature is invalid: -viewer.utilityPane.signatures.tab.certTree.cert.unknown.label=Signature is valid: -viewer.utilityPane.signatures.tab.certTree.cert.valid.label=Signature validity is unknown: -viewer.utilityPane.signatures.tab.certTree.doc.modified.label=This version of the document is unaltered but subsequent changes have been made -viewer.utilityPane.signatures.tab.certTree.doc.unmodified.label=Document has not been modified since it was signed -viewer.utilityPane.signatures.tab.certTree.doc.major.label=Document has been altered or corrupted since it was signed -viewer.utilityPane.signatures.tab.certTree.signature.identity.unknown.label=Signer's identity is unknown because it could not be found in your keystore -viewer.utilityPane.signatures.tab.certTree.signature.identity.unchecked.label=Signature is valid, but revocation of the signer's identity could not be checked -viewer.utilityPane.signatures.tab.certTree.signature.identity.valid.label=Signer's identity is valid -viewer.utilityPane.signatures.tab.certTree.signature.time.local.label=Signing time is from the clock on this signer's computer -viewer.utilityPane.signatures.tab.certTree.signature.time.embedded.label=Signature included an embedded timestamp but it could not be validated -viewer.utilityPane.signatures.tab.certTree.signature.details.label=Signature Details -viewer.utilityPane.signatures.tab.certTree.signature.details.reason.label=Reason: {0} -viewer.utilityPane.signatures.tab.certTree.signature.details.location.label=Location: {0} -viewer.utilityPane.signatures.tab.certTree.signature.details.full.label=Certificate Details... -viewer.utilityPane.signatures.tab.certTree.signature.lastChecked.label=Last Checked: {0} -viewer.utilityPane.signatures.tab.certTree.unsigned.label=Unsigned Signature Fields +viewer.utilityPane.signatures.tab.title = Signatures +viewer.utilityPane.signatures.tab.certTree.error.label = \ + Unsigned Signature Fields Signer certificate could not be validated {0} {1} +viewer.utilityPane.signatures.tab.certTree.rootSigned.label = Signed by {0} {1} +viewer.utilityPane.signatures.tab.certTree.rootValidating.label = Validating signature {0} {1} +viewer.utilityPane.signatures.tab.certTree.cert.invalid.label = Signature is invalid: +viewer.utilityPane.signatures.tab.certTree.cert.unknown.label = Signature is valid: +viewer.utilityPane.signatures.tab.certTree.cert.valid.label = Signature validity is unknown: +viewer.utilityPane.signatures.tab.certTree.doc.modified.label = \ + This version of the document is unaltered but subsequent changes have been made +viewer.utilityPane.signatures.tab.certTree.doc.unmodified.label = Document has not been modified since it was signed +viewer.utilityPane.signatures.tab.certTree.doc.major.label = Document has been altered or corrupted since it was signed +viewer.utilityPane.signatures.tab.certTree.signature.identity.unknown.label = \ + Signer's identity is unknown because it could not be found in your keystore +viewer.utilityPane.signatures.tab.certTree.signature.identity.unchecked.label = \ + Signature is valid, but revocation of the signer's identity could not be checked +viewer.utilityPane.signatures.tab.certTree.signature.identity.valid.label = Signer's identity is valid +viewer.utilityPane.signatures.tab.certTree.signature.time.local.label = Signing time is from the clock on this signer's computer +viewer.utilityPane.signatures.tab.certTree.signature.time.embedded.label = \ + Signature included an embedded timestamp but it could not be validated +viewer.utilityPane.signatures.tab.certTree.signature.details.label = Signature Details +viewer.utilityPane.signatures.tab.certTree.signature.details.reason.label = Reason: {0} +viewer.utilityPane.signatures.tab.certTree.signature.details.location.label = Location: {0} +viewer.utilityPane.signatures.tab.certTree.signature.details.full.label = Certificate Details... +viewer.utilityPane.signatures.tab.certTree.signature.lastChecked.label = Last Checked: {0} +viewer.utilityPane.signatures.tab.certTree.unsigned.label = Unsigned Signature Fields ## Signature certificate view dialog. -viewer.utilityPane.signatures.cert.dialog.title=Certificate Details -viewer.utilityPane.signatures.cert.dialog.closeButton.label=Close -viewer.utilityPane.signatures.cert.dialog.closeButton.mnemonic=C -viewer.utilityPane.signatures.cert.dialog.info.notAvailable.label=N/A -viewer.utilityPane.signatures.cert.dialog.info.unknownSubject.label=N/A Subject -viewer.utilityPane.signatures.cert.dialog.info.unknownIssuer.label=N/A Issuer -viewer.utilityPane.signatures.cert.dialog.info.certificateInfo.label={0} - {1} -viewer.utilityPane.signatures.cert.dialog.info.column1.label=Field -viewer.utilityPane.signatures.cert.dialog.info.column2.label=Value -viewer.utilityPane.signatures.cert.dialog.info.version.label=Version -viewer.utilityPane.signatures.cert.dialog.info.version.value=v{0} -viewer.utilityPane.signatures.cert.dialog.info.serialNumber.label=Serial Number -viewer.utilityPane.signatures.cert.dialog.info.serialNumber.value={0} -viewer.utilityPane.signatures.cert.dialog.info.signatureAlgorithm.label=Signature Algorithm -viewer.utilityPane.signatures.cert.dialog.info.signatureAlgorithm.value={0} -viewer.utilityPane.signatures.cert.dialog.info.issuer.label=Issuer -viewer.utilityPane.signatures.cert.dialog.info.issuer.value=Organization: {0} \nOrganization Unit: {1} \nCommon Name: {2} \nLocal: {3} \nState: {4} \nCountry: {5} \nEmail: {6} -viewer.utilityPane.signatures.cert.dialog.info.validity.label=Validity -viewer.utilityPane.signatures.cert.dialog.info.validity.value=From: {0}\n To: {1} -viewer.utilityPane.signatures.cert.dialog.info.subject.label=Subject -viewer.utilityPane.signatures.cert.dialog.info.subject.value=Organization: {0} \nOrganization Unit: {1} \nCommon Name: {2} \nLocal: {3} \nState: {4} \nCountry: {5} \nEmail: {6} -viewer.utilityPane.signatures.cert.dialog.info.signature.label=Signature -viewer.utilityPane.signatures.cert.dialog.info.md5.label=MD5 Fingerprint -viewer.utilityPane.signatures.cert.dialog.info.md5.value={0} -viewer.utilityPane.signatures.cert.dialog.info.sha1.label=SHA1 Fingerprint -viewer.utilityPane.signatures.cert.dialog.info.sha1.value={0} -viewer.utilityPane.signatures.verify.initializingMessage.label=Validating {0} of {1} Signatures -viewer.utilityPane.signatures.verify.completeMessage.label=Validating process complete -viewer.utilityPane.signatures.verify.validating.label=Validating signature... +viewer.utilityPane.signatures.cert.dialog.title = Certificate Details +viewer.utilityPane.signatures.cert.dialog.closeButton.label = Close +viewer.utilityPane.signatures.cert.dialog.closeButton.mnemonic = C +viewer.utilityPane.signatures.cert.dialog.info.notAvailable.label = N/A +viewer.utilityPane.signatures.cert.dialog.info.unknownSubject.label = N/A Subject +viewer.utilityPane.signatures.cert.dialog.info.unknownIssuer.label = N/A Issuer +viewer.utilityPane.signatures.cert.dialog.info.certificateInfo.label = {0} - {1} +viewer.utilityPane.signatures.cert.dialog.info.column1.label = Field +viewer.utilityPane.signatures.cert.dialog.info.column2.label = Value +viewer.utilityPane.signatures.cert.dialog.info.version.label = Version +viewer.utilityPane.signatures.cert.dialog.info.version.value = v{0} +viewer.utilityPane.signatures.cert.dialog.info.serialNumber.label = Serial Number +viewer.utilityPane.signatures.cert.dialog.info.serialNumber.value = {0} +viewer.utilityPane.signatures.cert.dialog.info.signatureAlgorithm.label = Signature Algorithm +viewer.utilityPane.signatures.cert.dialog.info.signatureAlgorithm.value = {0} +viewer.utilityPane.signatures.cert.dialog.info.issuer.label = Issuer +viewer.utilityPane.signatures.cert.dialog.info.issuer.value = \ + Organization: {0} \nOrganization Unit: {1} \nCommon Name: {2} \nLocal: {3} \nState: {4} \nCountry: {5} \nEmail: {6} +viewer.utilityPane.signatures.cert.dialog.info.validity.label = Validity +viewer.utilityPane.signatures.cert.dialog.info.validity.value = From: {0}\n To: {1} +viewer.utilityPane.signatures.cert.dialog.info.subject.label = Subject +viewer.utilityPane.signatures.cert.dialog.info.subject.value = \ + Organization: {0} \nOrganization Unit: {1} \nCommon Name: {2} \nLocal: {3} \nState: {4} \nCountry: {5} \nEmail: {6} +viewer.utilityPane.signatures.cert.dialog.info.signature.label = Signature +viewer.utilityPane.signatures.cert.dialog.info.md5.label = MD5 Fingerprint +viewer.utilityPane.signatures.cert.dialog.info.md5.value = {0} +viewer.utilityPane.signatures.cert.dialog.info.sha1.label = SHA1 Fingerprint +viewer.utilityPane.signatures.cert.dialog.info.sha1.value = {0} +viewer.utilityPane.signatures.verify.initializingMessage.label = Validating {0} of {1} Signatures +viewer.utilityPane.signatures.verify.completeMessage.label = Validating process complete +viewer.utilityPane.signatures.verify.validating.label = Validating signature... ## Annotation Tab -viewer.utilityPane.annotation.tab.title=Annotations +viewer.utilityPane.annotation.tab.title = Annotations ## Utility Pane Annotation Link Tab -viewer.utilityPane.annotation.link.appearance.title=Link Annotation -viewer.utilityPane.annotation.link.highlightType=Highlight Style: -viewer.utilityPane.annotation.link.none=None -viewer.utilityPane.annotation.link.invert=Invert` -viewer.utilityPane.annotation.link.outline=Outline -viewer.utilityPane.annotation.link.push=Push +viewer.utilityPane.annotation.link.appearance.title = Link Annotation +viewer.utilityPane.annotation.link.highlightType = Highlight Style: +viewer.utilityPane.annotation.link.none = None +viewer.utilityPane.annotation.link.invert = Invert` +viewer.utilityPane.annotation.link.outline = Outline +viewer.utilityPane.annotation.link.push = Push ## Utility Pane Annotation text markup Tab -viewer.utilityPane.annotation.textMarkup.appearance.title=Text Markup Annotation -viewer.utilityPane.annotation.textMarkup.highlightType=Type: -viewer.utilityPane.annotation.textMarkup.colorChooserTitle=MarKup Color -viewer.utilityPane.annotation.textMarkup.colorLabel=Color: -viewer.utilityPane.annotation.textMarkup.transparencyLabel=Transparency: +viewer.utilityPane.annotation.textMarkup.appearance.title = Text Markup Annotation +viewer.utilityPane.annotation.textMarkup.highlightType = Type: +viewer.utilityPane.annotation.textMarkup.colorChooserTitle = MarKup Color +viewer.utilityPane.annotation.textMarkup.colorLabel = Color: +viewer.utilityPane.annotation.textMarkup.transparencyLabel = Transparency: ## Utility Pane Annotation line Tab -viewer.utilityPane.annotation.line.appearance.title=Line Annotation -viewer.utilityPane.annotation.line.lineThickness=Line Thickness: -viewer.utilityPane.annotation.line.lineStyle=Line Style: -viewer.utilityPane.annotation.line.startStyle=Start: -viewer.utilityPane.annotation.line.endStyle=End: -viewer.utilityPane.annotation.line.colorChooserTitle=Line Color -viewer.utilityPane.annotation.line.colorInternalChooserTitle=Line Internal Color -viewer.utilityPane.annotation.line.colorLabel=Color: -viewer.utilityPane.annotation.line.colorInternalLabel=Fill Color: -viewer.utilityPane.annotation.line.end.none=None -viewer.utilityPane.annotation.line.end.openArrow=Open Arrow -viewer.utilityPane.annotation.line.end.closedArrow=Closed Arrow -viewer.utilityPane.annotation.line.end.diamond=Diamond -viewer.utilityPane.annotation.line.end.square=Square -viewer.utilityPane.annotation.line.end.circle=Circle -viewer.utilityPane.annotation.line.transparencyLabel=Transparency: +viewer.utilityPane.annotation.line.appearance.title = Line Annotation +viewer.utilityPane.annotation.line.lineThickness = Line Thickness: +viewer.utilityPane.annotation.line.lineStyle = Line Style: +viewer.utilityPane.annotation.line.startStyle = Start: +viewer.utilityPane.annotation.line.endStyle = End: +viewer.utilityPane.annotation.line.colorChooserTitle = Line Color +viewer.utilityPane.annotation.line.colorInternalChooserTitle = Line Internal Color +viewer.utilityPane.annotation.line.colorLabel = Color: +viewer.utilityPane.annotation.line.colorInternalLabel = Fill Color: +viewer.utilityPane.annotation.line.end.none = None +viewer.utilityPane.annotation.line.end.openArrow = Open Arrow +viewer.utilityPane.annotation.line.end.closedArrow = Closed Arrow +viewer.utilityPane.annotation.line.end.diamond = Diamond +viewer.utilityPane.annotation.line.end.square = Square +viewer.utilityPane.annotation.line.end.circle = Circle +viewer.utilityPane.annotation.line.transparencyLabel = Transparency: ## Utility Pane Annotation square Tab -viewer.utilityPane.annotation.square.appearance.title=Square Annotation -viewer.utilityPane.annotation.square.lineThickness=Border Thickness: -viewer.utilityPane.annotation.square.lineStyle=Border Style: -viewer.utilityPane.annotation.square.colorBorderChooserTitle=Border Color -viewer.utilityPane.annotation.square.colorInteriorChooserTitle=Fill Color -viewer.utilityPane.annotation.square.borderTypeLabel=Border Type: -viewer.utilityPane.annotation.square.colorBorderLabel=Border Color: -viewer.utilityPane.annotation.square.colorInteriorLabel=Fill Color: -viewer.utilityPane.annotation.square.fillTypeLabel=Fill Type: -viewer.utilityPane.annotation.square.transparencyLabel=Transparency: +viewer.utilityPane.annotation.square.appearance.title = Square Annotation +viewer.utilityPane.annotation.square.lineThickness = Border Thickness: +viewer.utilityPane.annotation.square.lineStyle = Border Style: +viewer.utilityPane.annotation.square.colorBorderChooserTitle = Border Color +viewer.utilityPane.annotation.square.colorInteriorChooserTitle = Fill Color +viewer.utilityPane.annotation.square.borderTypeLabel = Border Type: +viewer.utilityPane.annotation.square.colorBorderLabel = Border Color: +viewer.utilityPane.annotation.square.colorInteriorLabel = Fill Color: +viewer.utilityPane.annotation.square.fillTypeLabel = Fill Type: +viewer.utilityPane.annotation.square.transparencyLabel = Transparency: ## Utility Pane Annotation free text Tab -viewer.utilityPane.annotation.freeText.appearance.title=FreeText Annotation -viewer.utilityPane.annotation.freeText.font.name=Font Name: -viewer.utilityPane.annotation.freeText.font.style=Font Style: -viewer.utilityPane.annotation.freeText.font.size=Font Size: -viewer.utilityPane.annotation.freeText.font.color=Font Color: -viewer.utilityPane.annotation.freeText.font.color.ChooserTitle=Font Color -viewer.utilityPane.annotation.freeText.border.thickness=Border Thickness: -viewer.utilityPane.annotation.freeText.border.type=Border Type: -viewer.utilityPane.annotation.freeText.border.style=Border Style: -viewer.utilityPane.annotation.freeText.border.color=Border Color: -viewer.utilityPane.annotation.freeText.border.color.ChooserTitle=Border Color -viewer.utilityPane.annotation.freeText.fill.type=Fill Type: -viewer.utilityPane.annotation.freeText.fill.color=Fill Color: -viewer.utilityPane.annotation.freeText.transparencyLabel=Transparency: -viewer.utilityPane.annotation.freeText.fill.color.ChooserTitle=Fill Color -viewer.utilityPane.annotation.freeText.font.dialog=Dialog -viewer.utilityPane.annotation.freeText.font.dialogInput=DialogInput -viewer.utilityPane.annotation.freeText.font.monospaced=Monospaced -viewer.utilityPane.annotation.freeText.font.serif=Serif -viewer.utilityPane.annotation.freeText.font.sanSerif=SansSerif -viewer.utilityPane.annotation.freeText.font.style.plain=Plain -viewer.utilityPane.annotation.freeText.font.style.italic=Italic -viewer.utilityPane.annotation.freeText.font.style.bold=Bold -viewer.utilityPane.annotation.freeText.font.name.helvetica=Helvetica -viewer.utilityPane.annotation.freeText.font.name.helveticaOblique=Helvetica-Oblique -viewer.utilityPane.annotation.freeText.font.name.helveticaBold=Helvetica-Bold -viewer.utilityPane.annotation.freeText.font.name.HelveticaBoldOblique=Helvetica-BoldOblique -viewer.utilityPane.annotation.freeText.font.name.timesItalic=Times-Italic -viewer.utilityPane.annotation.freeText.font.name.timesBold=Times-Bold -viewer.utilityPane.annotation.freeText.font.name.timesBoldItalic=Times-BoldItalic -viewer.utilityPane.annotation.freeText.font.name.timesRoman=Times-Roman -viewer.utilityPane.annotation.freeText.font.name.courier=Courier -viewer.utilityPane.annotation.freeText.font.name.courierOblique=Courier-Oblique -viewer.utilityPane.annotation.freeText.font.name.courierBoldOblique=Courier-BoldOblique -viewer.utilityPane.annotation.freeText.font.name.courierBold=Courier-Bold +viewer.utilityPane.annotation.freeText.appearance.title = FreeText Annotation +viewer.utilityPane.annotation.freeText.font.name = Font Name: +viewer.utilityPane.annotation.freeText.font.style = Font Style: +viewer.utilityPane.annotation.freeText.font.size = Font Size: +viewer.utilityPane.annotation.freeText.font.color = Font Color: +viewer.utilityPane.annotation.freeText.font.color.ChooserTitle = Font Color +viewer.utilityPane.annotation.freeText.border.thickness = Border Thickness: +viewer.utilityPane.annotation.freeText.border.type = Border Type: +viewer.utilityPane.annotation.freeText.border.style = Border Style: +viewer.utilityPane.annotation.freeText.border.color = Border Color: +viewer.utilityPane.annotation.freeText.border.color.ChooserTitle = Border Color +viewer.utilityPane.annotation.freeText.fill.type = Fill Type: +viewer.utilityPane.annotation.freeText.fill.color = Fill Color: +viewer.utilityPane.annotation.freeText.transparencyLabel = Transparency: +viewer.utilityPane.annotation.freeText.fill.color.ChooserTitle = Fill Color +viewer.utilityPane.annotation.freeText.font.dialog = Dialog +viewer.utilityPane.annotation.freeText.font.dialogInput = DialogInput +viewer.utilityPane.annotation.freeText.font.monospaced = Monospaced +viewer.utilityPane.annotation.freeText.font.serif = Serif +viewer.utilityPane.annotation.freeText.font.sanSerif = SansSerif +viewer.utilityPane.annotation.freeText.font.style.plain = Plain +viewer.utilityPane.annotation.freeText.font.style.italic = Italic +viewer.utilityPane.annotation.freeText.font.style.bold = Bold +viewer.utilityPane.annotation.freeText.font.name.helvetica = Helvetica +viewer.utilityPane.annotation.freeText.font.name.helveticaOblique = Helvetica-Oblique +viewer.utilityPane.annotation.freeText.font.name.helveticaBold = Helvetica-Bold +viewer.utilityPane.annotation.freeText.font.name.HelveticaBoldOblique = Helvetica-BoldOblique +viewer.utilityPane.annotation.freeText.font.name.timesItalic = Times-Italic +viewer.utilityPane.annotation.freeText.font.name.timesBold = Times-Bold +viewer.utilityPane.annotation.freeText.font.name.timesBoldItalic = Times-BoldItalic +viewer.utilityPane.annotation.freeText.font.name.timesRoman = Times-Roman +viewer.utilityPane.annotation.freeText.font.name.courier = Courier +viewer.utilityPane.annotation.freeText.font.name.courierOblique = Courier-Oblique +viewer.utilityPane.annotation.freeText.font.name.courierBoldOblique = Courier-BoldOblique +viewer.utilityPane.annotation.freeText.font.name.courierBold = Courier-Bold ## Utility Pane Annotation text Tab -viewer.utilityPane.annotation.text.appearance.title=Text Annotation -viewer.utilityPane.annotation.text.iconName=Icon: -viewer.utilityPane.annotation.text.iconName.comment=Comment -viewer.utilityPane.annotation.text.iconName.check=Check -viewer.utilityPane.annotation.text.iconName.checkMark=CheckMark -viewer.utilityPane.annotation.text.iconName.circle=Circle -viewer.utilityPane.annotation.text.iconName.cross=Cross -viewer.utilityPane.annotation.text.iconName.crossHairs=CrossHairs -viewer.utilityPane.annotation.text.iconName.help=Help -viewer.utilityPane.annotation.text.iconName.insert=Insert -viewer.utilityPane.annotation.text.iconName.key=Key -viewer.utilityPane.annotation.text.iconName.newParagraph=NewParagraph -viewer.utilityPane.annotation.text.iconName.paragraph=Paragraph -viewer.utilityPane.annotation.text.iconName.rightArrow=RightArrow -viewer.utilityPane.annotation.text.iconName.rightPointer=RightPointer -viewer.utilityPane.annotation.text.iconName.star=Star -viewer.utilityPane.annotation.text.iconName.upArrow=UpArrow -viewer.utilityPane.annotation.text.iconName.upLeftArrow=UpLeftArrow +viewer.utilityPane.annotation.text.appearance.title = Text Annotation +viewer.utilityPane.annotation.text.iconName = Icon: +viewer.utilityPane.annotation.text.iconName.comment = Comment +viewer.utilityPane.annotation.text.iconName.check = Check +viewer.utilityPane.annotation.text.iconName.checkMark = CheckMark +viewer.utilityPane.annotation.text.iconName.circle = Circle +viewer.utilityPane.annotation.text.iconName.cross = Cross +viewer.utilityPane.annotation.text.iconName.crossHairs = CrossHairs +viewer.utilityPane.annotation.text.iconName.help = Help +viewer.utilityPane.annotation.text.iconName.insert = Insert +viewer.utilityPane.annotation.text.iconName.key = Key +viewer.utilityPane.annotation.text.iconName.newParagraph = NewParagraph +viewer.utilityPane.annotation.text.iconName.paragraph = Paragraph +viewer.utilityPane.annotation.text.iconName.rightArrow = RightArrow +viewer.utilityPane.annotation.text.iconName.rightPointer = RightPointer +viewer.utilityPane.annotation.text.iconName.star = Star +viewer.utilityPane.annotation.text.iconName.upArrow = UpArrow +viewer.utilityPane.annotation.text.iconName.upLeftArrow = UpLeftArrow ## Utility Pane Annotation circle Tab -viewer.utilityPane.annotation.circle.appearance.title=Circle Annotation -viewer.utilityPane.annotation.circle.lineThickness=Border Thickness: -viewer.utilityPane.annotation.circle.lineStyle=Border Style: -viewer.utilityPane.annotation.circle.colorBorderChooserTitle=Border Color -viewer.utilityPane.annotation.circle.colorInteriorChooserTitle=Interior Color -viewer.utilityPane.annotation.circle.colorBorderLabel=Border Color: -viewer.utilityPane.annotation.circle.colorInteriorLabel=Fill Color: -viewer.utilityPane.annotation.circle.fillTypeLabel=Fill Type: -viewer.utilityPane.annotation.circle.transparencyLabel=Transparency: +viewer.utilityPane.annotation.circle.appearance.title = Circle Annotation +viewer.utilityPane.annotation.circle.lineThickness = Border Thickness: +viewer.utilityPane.annotation.circle.lineStyle = Border Style: +viewer.utilityPane.annotation.circle.colorBorderChooserTitle = Border Color +viewer.utilityPane.annotation.circle.colorInteriorChooserTitle = Interior Color +viewer.utilityPane.annotation.circle.colorBorderLabel = Border Color: +viewer.utilityPane.annotation.circle.colorInteriorLabel = Fill Color: +viewer.utilityPane.annotation.circle.fillTypeLabel = Fill Type: +viewer.utilityPane.annotation.circle.transparencyLabel = Transparency: ## Utility Pane Annotation ink Tab -viewer.utilityPane.annotation.ink.appearance.title=Ink Annotation -viewer.utilityPane.annotation.ink.lineThickness=Ink Thickness: -viewer.utilityPane.annotation.ink.lineStyle=Ink Style: -viewer.utilityPane.annotation.ink.colorBorderChooserTitle=Ink Color -viewer.utilityPane.annotation.ink.colorBorderLabel=Ink Color: -viewer.utilityPane.annotation.ink.transparencyLabel=Transparency: +viewer.utilityPane.annotation.ink.appearance.title = Ink Annotation +viewer.utilityPane.annotation.ink.lineThickness = Ink Thickness: +viewer.utilityPane.annotation.ink.lineStyle = Ink Style: +viewer.utilityPane.annotation.ink.colorBorderChooserTitle = Ink Color +viewer.utilityPane.annotation.ink.colorBorderLabel = Ink Color: +viewer.utilityPane.annotation.ink.transparencyLabel = Transparency: ## Utility Pane border Tab -viewer.utilityPane.annotation.border.title=Border -viewer.utilityPane.annotation.border.linkType=Border Type: -viewer.utilityPane.annotation.border.lineThickness=Border Thickness: -viewer.utilityPane.annotation.border.lineStyle=Border Style: -viewer.utilityPane.annotation.border.colorChooserTitle=Border Color -viewer.utilityPane.annotation.border.colorLabel=Color: -viewer.utilityPane.annotation.border.borderType.visibleRectangle=Visible -viewer.utilityPane.annotation.border.borderType.invisibleRectangle=Invisible -viewer.utilityPane.annotation.border.solid=Solid -viewer.utilityPane.annotation.border.dashed=Dashed -viewer.utilityPane.annotation.border.beveled=Beveled -viewer.utilityPane.annotation.border.inset=Inset -viewer.utilityPane.annotation.border.underline=Underline +viewer.utilityPane.annotation.border.title = Border +viewer.utilityPane.annotation.border.linkType = Border Type: +viewer.utilityPane.annotation.border.lineThickness = Border Thickness: +viewer.utilityPane.annotation.border.lineStyle = Border Style: +viewer.utilityPane.annotation.border.colorChooserTitle = Border Color +viewer.utilityPane.annotation.border.colorLabel = Color: +viewer.utilityPane.annotation.border.borderType.visibleRectangle = Visible +viewer.utilityPane.annotation.border.borderType.invisibleRectangle = Invisible +viewer.utilityPane.annotation.border.solid = Solid +viewer.utilityPane.annotation.border.dashed = Dashed +viewer.utilityPane.annotation.border.beveled = Beveled +viewer.utilityPane.annotation.border.inset = Inset +viewer.utilityPane.annotation.border.underline = Underline ## Utility Pane border Tab -viewer.utilityPane.annotation.flags.title=Flags -viewer.utilityPane.annotation.flags.noRotate=No Rotate: -viewer.utilityPane.annotation.flags.noZoom=No Zoom: -viewer.utilityPane.annotation.flags.readOnly=Read Only: -viewer.utilityPane.annotation.flags.printable=Printable: -viewer.utilityPane.annotation.flags.yes=Printable: -viewer.utilityPane.annotation.flags.enabled=Enabled -viewer.utilityPane.annotation.flags.disabled=Disabled +viewer.utilityPane.annotation.flags.title = Flags +viewer.utilityPane.annotation.flags.noRotate = No Rotate: +viewer.utilityPane.annotation.flags.noZoom = No Zoom: +viewer.utilityPane.annotation.flags.readOnly = Read Only: +viewer.utilityPane.annotation.flags.printable = Printable: +viewer.utilityPane.annotation.flags.yes = Printable: +viewer.utilityPane.annotation.flags.enabled = Enabled +viewer.utilityPane.annotation.flags.disabled = Disabled ## annotation action pane and dialogs. -viewer.utilityPane.action.selectionTitle=Action -viewer.utilityPane.action.addAction=Add -viewer.utilityPane.action.editAction=Edit -viewer.utilityPane.action.removeAction=Remove -viewer.utilityPane.action.type.destination.label=Destination -viewer.utilityPane.action.type.uriAction.label=URI Action -viewer.utilityPane.action.type.goToAction.label=GoTo Action -viewer.utilityPane.action.type.launchAction.label=Launch Action -viewer.utilityPane.action.dialog.new.title=Add New Action -viewer.utilityPane.action.dialog.new.msgs=Action Type: -viewer.utilityPane.action.dialog.delete.title=Delete Confirmation -viewer.utilityPane.action.dialog.delete.msgs=Are you sure your want to delete this action? +viewer.utilityPane.action.selectionTitle = Action +viewer.utilityPane.action.addAction = Add +viewer.utilityPane.action.editAction = Edit +viewer.utilityPane.action.removeAction = Remove +viewer.utilityPane.action.type.destination.label = Destination +viewer.utilityPane.action.type.uriAction.label = URI Action +viewer.utilityPane.action.type.goToAction.label = GoTo Action +viewer.utilityPane.action.type.launchAction.label = Launch Action +viewer.utilityPane.action.dialog.new.title = Add New Action +viewer.utilityPane.action.dialog.new.msgs = Action Type: +viewer.utilityPane.action.dialog.delete.title = Delete Confirmation +viewer.utilityPane.action.dialog.delete.msgs = Are you sure your want to delete this action? ## uri action dialog test -viewer.utilityPane.action.dialog.uri.title=URI Action Properties -viewer.utilityPane.action.dialog.uri.msgs=URI: +viewer.utilityPane.action.dialog.uri.title = URI Action Properties +viewer.utilityPane.action.dialog.uri.msgs = URI: ## launch action dialog test -viewer.utilityPane.action.dialog.launch.title=Launch Action Properties -viewer.utilityPane.action.dialog.launch.msgs=File Path: +viewer.utilityPane.action.dialog.launch.title = Launch Action Properties +viewer.utilityPane.action.dialog.launch.msgs = File Path: ## GoTo action dialog text -viewer.utilityPane.action.dialog.goto.title=GoTo Action Properties -viewer.utilityPane.action.dialog.goto.page.label=Page: -viewer.utilityPane.action.dialog.goto.type.label=Type -viewer.utilityPane.action.dialog.goto.type.xyz.label=Absolute -viewer.utilityPane.action.dialog.goto.type.fit.label=Fit Page -viewer.utilityPane.action.dialog.goto.type.fith.label=Fit Top Width -viewer.utilityPane.action.dialog.goto.type.fitv.label=Fit Left Width -viewer.utilityPane.action.dialog.goto.type.fitr.label=Fit Zoom Box -viewer.utilityPane.action.dialog.goto.type.fitb.label=Fit Page Bounds -viewer.utilityPane.action.dialog.goto.type.fitbh.label=Fit Bounds Top -viewer.utilityPane.action.dialog.goto.type.fitbv.label=Fit Bounds Left -viewer.utilityPane.action.dialog.goto.right.label=Right: -viewer.utilityPane.action.dialog.goto.left.label=Left: -viewer.utilityPane.action.dialog.goto.top.label=Top: -viewer.utilityPane.action.dialog.goto.bottom.label=Bottom: -viewer.utilityPane.action.dialog.goto.zoom.label=Zoom: -viewer.utilityPane.action.dialog.goto.unassigned.label=NaN -viewer.utilityPane.action.dialog.goto.current.label=Current View: -viewer.utilityPane.action.dialog.goto.current=Set Location -viewer.utilityPane.action.dialog.goto.name.label=Name: -viewer.utilityPane.action.dialog.goto.browse=Browse... -viewer.utilityPane.action.dialog.goto.explicitDestination.title=Implicit Destination -viewer.utilityPane.action.dialog.goto.nameDestination.title=Named Destination +viewer.utilityPane.action.dialog.goto.title = GoTo Action Properties +viewer.utilityPane.action.dialog.goto.page.label = Page: +viewer.utilityPane.action.dialog.goto.type.label = Type +viewer.utilityPane.action.dialog.goto.type.xyz.label = Absolute +viewer.utilityPane.action.dialog.goto.type.fit.label = Fit Page +viewer.utilityPane.action.dialog.goto.type.fith.label = Fit Top Width +viewer.utilityPane.action.dialog.goto.type.fitv.label = Fit Left Width +viewer.utilityPane.action.dialog.goto.type.fitr.label = Fit Zoom Box +viewer.utilityPane.action.dialog.goto.type.fitb.label = Fit Page Bounds +viewer.utilityPane.action.dialog.goto.type.fitbh.label = Fit Bounds Top +viewer.utilityPane.action.dialog.goto.type.fitbv.label = Fit Bounds Left +viewer.utilityPane.action.dialog.goto.right.label = Right: +viewer.utilityPane.action.dialog.goto.left.label = Left: +viewer.utilityPane.action.dialog.goto.top.label = Top: +viewer.utilityPane.action.dialog.goto.bottom.label = Bottom: +viewer.utilityPane.action.dialog.goto.zoom.label = Zoom: +viewer.utilityPane.action.dialog.goto.unassigned.label = NaN +viewer.utilityPane.action.dialog.goto.current.label = Current View: +viewer.utilityPane.action.dialog.goto.current = Set Location +viewer.utilityPane.action.dialog.goto.name.label = Name: +viewer.utilityPane.action.dialog.goto.browse = Browse... +viewer.utilityPane.action.dialog.goto.explicitDestination.title = Implicit Destination +viewer.utilityPane.action.dialog.goto.nameDestination.title = Named Destination # Destination Named Tree -viewer.utilityPane.action.dialog.goto.nameTree.title=Document Name Tree -viewer.utilityPane.action.dialog.goto.nameTree.root.label=Name Tree -viewer.utilityPane.action.dialog.goto.nameTree.branch.label={0} to {1} +viewer.utilityPane.action.dialog.goto.nameTree.title = Document Name Tree +viewer.utilityPane.action.dialog.goto.nameTree.root.label = Name Tree +viewer.utilityPane.action.dialog.goto.nameTree.branch.label = {0} to {1} ## Utility Pane Search Tab -viewer.utilityPane.search.tab.title=Search -viewer.utilityPane.search.searchText.label=Search Text: -viewer.utilityPane.search.results.label=Results: -viewer.utilityPane.search.searchButton.label=Search -viewer.utilityPane.search.clearSearchButton.label=Clear -viewer.utilityPane.search.caseSenstiveCheckbox.label=Case-sensitive -viewer.utilityPane.search.wholeWordCheckbox.label=Whole words only -viewer.utilityPane.search.cumlitiveCheckbox.label=Cumulative -viewer.utilityPane.search.showPagesCheckbox.label=Show Pages -viewer.utilityPane.search.stopButton.label=Stop -viewer.utilityPane.search.searching.msg=Search... +viewer.utilityPane.search.tab.title = Search +viewer.utilityPane.search.searchText.label = Search Text: +viewer.utilityPane.search.results.label = Results: +viewer.utilityPane.search.searchButton.label = Search +viewer.utilityPane.search.clearSearchButton.label = Clear +viewer.utilityPane.search.caseSenstiveCheckbox.label = Case-sensitive +viewer.utilityPane.search.wholeWordCheckbox.label = Whole words only +viewer.utilityPane.search.cumlitiveCheckbox.label = Cumulative +viewer.utilityPane.search.showPagesCheckbox.label = Show Pages +viewer.utilityPane.search.stopButton.label = Stop +viewer.utilityPane.search.searching.msg = Search... # Searching x out of y page(s) -viewer.utilityPane.search.searching1.msg=Searching {0} out of {1} -viewer.utilityPane.search.searching1.oneFile.msg={2} page -viewer.utilityPane.search.searching1.moreFile.msg={2} pages +viewer.utilityPane.search.searching1.msg = \ + Searching {0} out of {1} +viewer.utilityPane.search.searching1.oneFile.msg = {2} page +viewer.utilityPane.search.searching1.moreFile.msg = {2} pages # Page x (y result(s)) -viewer.utilityPane.search.result.msg=Page {0} ({1}) -viewer.utilityPane.search.result.oneFile.msg={2} result -viewer.utilityPane.search.result.moreFile.msg={2} results +viewer.utilityPane.search.result.msg = Page {0} ({1}) +viewer.utilityPane.search.result.oneFile.msg = {2} result +viewer.utilityPane.search.result.moreFile.msg = {2} results # Searched x page(s) (y matches) -viewer.utilityPane.search.progress.msg=Searched {0} {1} ({2}) -viewer.utilityPane.search.progress.onePage.msg=page -viewer.utilityPane.search.progress.morePage.msg=pages -viewer.utilityPane.search.progress.oneMatch.msg={2} match -viewer.utilityPane.search.progress.moreMatch.msg={2} matches +viewer.utilityPane.search.progress.msg = \ + Searched {0} {1} ({2}) +viewer.utilityPane.search.progress.onePage.msg = page +viewer.utilityPane.search.progress.morePage.msg = pages +viewer.utilityPane.search.progress.oneMatch.msg = {2} match +viewer.utilityPane.search.progress.moreMatch.msg = {2} matches ## Popup Annotation component -viewer.annotation.popup.reply.label=Reply -viewer.annotation.popup.delete.label=Delete -viewer.annotation.popup.status.label=Set Status -viewer.annotation.popup.status.accepted.label=Accepted -viewer.annotation.popup.status.cancelled.label=Cancelled -viewer.annotation.popup.status.completed.label=Completed -viewer.annotation.popup.status.rejected.label=Rejected -viewer.annotation.popup.status.none.label=None -viewer.annotation.popup.openAll.label=Open all Popups -viewer.annotation.popup.minimizeAll.label=Minimize Popups -viewer.annotation.popup.replyTo.label=Re: {0} -viewer.annotation.popup.status.none.title=None: {0} -viewer.annotation.popup.status.none.msg=None set by {0} -viewer.annotation.popup.status.accepted.title=Accepted: {0} -viewer.annotation.popup.status.accepted.msg=Accepted set by {0} -viewer.annotation.popup.status.cancelled.title=Cancelled: {0} -viewer.annotation.popup.status.cancelled.msg=Cancelled set by {0} -viewer.annotation.popup.status.completed.title=Completed: {0} -viewer.annotation.popup.status.completed.msg=Completed set by {0} -viewer.annotation.popup.status.rejected.title=Rejected: {0} -viewer.annotation.popup.status.rejected.msg=Rejected set by {0} +viewer.annotation.popup.reply.label = Reply +viewer.annotation.popup.delete.label = Delete +viewer.annotation.popup.status.label = Set Status +viewer.annotation.popup.status.accepted.label = Accepted +viewer.annotation.popup.status.cancelled.label = Cancelled +viewer.annotation.popup.status.completed.label = Completed +viewer.annotation.popup.status.rejected.label = Rejected +viewer.annotation.popup.status.none.label = None +viewer.annotation.popup.openAll.label = Open all Popups +viewer.annotation.popup.minimizeAll.label = Minimize Popups +viewer.annotation.popup.replyTo.label = Re: {0} +viewer.annotation.popup.status.none.title = None: {0} +viewer.annotation.popup.status.none.msg = None set by {0} +viewer.annotation.popup.status.accepted.title = Accepted: {0} +viewer.annotation.popup.status.accepted.msg = Accepted set by {0} +viewer.annotation.popup.status.cancelled.title = Cancelled: {0} +viewer.annotation.popup.status.cancelled.msg = Cancelled set by {0} +viewer.annotation.popup.status.completed.title = Completed: {0} +viewer.annotation.popup.status.completed.msg = Completed set by {0} +viewer.annotation.popup.status.rejected.title = Rejected: {0} +viewer.annotation.popup.status.rejected.msg = Rejected set by {0} ## Signature component -viewer.annotation.signature.menu.validateSignature.label=Validate Signature -viewer.annotation.signature.menu.showCertificates.label=Show Certificate Properties -viewer.annotation.signature.menu.signatureProperties.label=Show Signature Properties -viewer.annotation.signature.menu.signaturePageNavigation.label=Go to Page... +viewer.annotation.signature.menu.validateSignature.label = Validate Signature +viewer.annotation.signature.menu.showCertificates.label = Show Certificate Properties +viewer.annotation.signature.menu.signatureProperties.label = Show Signature Properties +viewer.annotation.signature.menu.signaturePageNavigation.label = Go to Page... ## Signature validation dialog. -viewer.annotation.signature.validation.dialog.title=Signature Validation Summary -viewer.annotation.signature.validation.dialog.close.button.label=Close -viewer.annotation.signature.validation.dialog.signerProperties.button.label=Signature Properties... +viewer.annotation.signature.validation.dialog.title = Signature Validation Summary +viewer.annotation.signature.validation.dialog.close.button.label = Close +viewer.annotation.signature.validation.dialog.signerProperties.button.label = Signature Properties... # common validation messages -viewer.annotation.signature.validation.common.invalid.label=Signature is invalid: -viewer.annotation.signature.validation.common.unknown.label=Signature is valid: -viewer.annotation.signature.validation.common.valid.label=Signature validity is unknown: -viewer.annotation.signature.validation.common.signedBy.label=- Signed by {0} {1} -viewer.annotation.signature.validation.common.doc.modified.label=- This version of the document is unaltered but subsequent changes have been made -viewer.annotation.signature.validation.common.doc.unmodified.label=- Document has not been modified since it was signed -viewer.annotation.signature.validation.common.doc.major.label=- Document has been altered or corrupted since it was signed -viewer.annotation.signature.validation.common.identity.unknown.label=- Signer's identity is unknown because it could not be found in your keystore -viewer.annotation.signature.validation.common.identity.unchecked.label=- Signature is valid, but revocation of the signer's identity could not be checked -viewer.annotation.signature.validation.common.identity.valid.label=- Signer's identity is valid -viewer.annotation.signature.validation.common.time.local.label=- Signing time is from the clock on this signer's computer -viewer.annotation.signature.validation.common.time.embedded.label=- Signature included an embedded timestamp but it could not be validated -viewer.annotation.signature.validation.common.notAvailable.label=N/A +viewer.annotation.signature.validation.common.invalid.label = Signature is invalid: +viewer.annotation.signature.validation.common.unknown.label = Signature is valid: +viewer.annotation.signature.validation.common.valid.label = Signature validity is unknown: +viewer.annotation.signature.validation.common.signedBy.label = - Signed by {0} {1} +viewer.annotation.signature.validation.common.doc.modified.label = \ + - This version of the document is unaltered but subsequent changes have been made +viewer.annotation.signature.validation.common.doc.unmodified.label = - Document has not been modified since it was signed +viewer.annotation.signature.validation.common.doc.major.label = - Document has been altered or corrupted since it was signed +viewer.annotation.signature.validation.common.identity.unknown.label = \ + - Signer's identity is unknown because it could not be found in your keystore +viewer.annotation.signature.validation.common.identity.unchecked.label = \ + - Signature is valid, but revocation of the signer's identity could not be checked +viewer.annotation.signature.validation.common.identity.valid.label = - Signer's identity is valid +viewer.annotation.signature.validation.common.time.local.label = - Signing time is from the clock on this signer's computer +viewer.annotation.signature.validation.common.time.embedded.label = \ + - Signature included an embedded timestamp but it could not be validated +viewer.annotation.signature.validation.common.notAvailable.label = N/A ## Signatures properties Dialog. -viewer.annotation.signature.properties.dialog.title=Signature Properties -viewer.annotation.signature.properties.dialog.invalid.label=Signature is invalid -viewer.annotation.signature.properties.dialog.unknown.label=Signature is valid -viewer.annotation.signature.properties.dialog.valid.label=Signature validity is unknown -viewer.annotation.signature.properties.dialog.signedBy.label=Signed by {0} {1} -viewer.annotation.signature.properties.dialog.signingTime.label=Signed time: {0} -viewer.annotation.signature.properties.dialog.reason.label=Reason: {0} -viewer.annotation.signature.properties.dialog.location.label=Location: {0} +viewer.annotation.signature.properties.dialog.title = Signature Properties +viewer.annotation.signature.properties.dialog.invalid.label = Signature is invalid +viewer.annotation.signature.properties.dialog.unknown.label = Signature is valid +viewer.annotation.signature.properties.dialog.valid.label = Signature validity is unknown +viewer.annotation.signature.properties.dialog.signedBy.label = Signed by {0} {1} +viewer.annotation.signature.properties.dialog.signingTime.label = Signed time: {0} +viewer.annotation.signature.properties.dialog.reason.label = Reason: {0} +viewer.annotation.signature.properties.dialog.location.label = Location: {0} # SignatureSigner Info -viewer.annotation.signature.properties.dialog.pathValidation.success=- Path validation checks were successful. -viewer.annotation.signature.properties.dialog.pathValidation.failure=- Path validation checks were unsuccessful. -viewer.annotation.signature.properties.dialog.revocation.success=- Signer's certificate is valid and has not been revoked. -viewer.annotation.signature.properties.dialog.revocation.failure=- Revocation checking was not performed. -viewer.annotation.signature.properties.dialog.certificateExpired.failure=- Signer certificate has expired. -viewer.annotation.signature.properties.dialog.showCertificates.label=Signer's Certificate... -viewer.annotation.signature.properties.dialog.validity.title=Validity Summary -viewer.annotation.signature.properties.dialog.signerInfo.title=Signer Info +viewer.annotation.signature.properties.dialog.pathValidation.success = - Path validation checks were successful. +viewer.annotation.signature.properties.dialog.pathValidation.failure = - Path validation checks were unsuccessful. +viewer.annotation.signature.properties.dialog.revocation.success = - Signer's certificate is valid and has not been revoked. +viewer.annotation.signature.properties.dialog.revocation.failure = - Revocation checking was not performed. +viewer.annotation.signature.properties.dialog.certificateExpired.failure = - Signer certificate has expired. +viewer.annotation.signature.properties.dialog.showCertificates.label = Signer's Certificate... +viewer.annotation.signature.properties.dialog.validity.title = Validity Summary +viewer.annotation.signature.properties.dialog.signerInfo.title = Signer Info ## Common Button Labels -viewer.button.ok.label=Ok -viewer.button.ok.mnemonic=O -viewer.button.cancel.label=Cancel -viewer.button.cancel.mnemonic=C +viewer.button.ok.label = Ok +viewer.button.ok.mnemonic = O +viewer.button.cancel.label = Cancel +viewer.button.cancel.mnemonic = C ## Pilot Specific Mesages -pilot.title=ICEbrowser - ICEpdf Pilot Errror -pilot.loading.msg=Opening document {0} ... -pilot.display.msg=Displaying {0} -pilot.loading.error.msg=PDF Pilot: Failed to load {0}. -pilot.error.classLoading=Required class {0} not found. Required library 'icepdf.jar' may not be on the classpath - PDF Pilot disabled."; +pilot.title = ICEbrowser - ICEpdf Pilot Errror +pilot.loading.msg =Opening document {0} ... +pilot.display.msg = Displaying {0} +pilot.loading.error.msg = PDF Pilot: Failed to load {0}. +pilot.error.classLoading = Required class {0} not found. Required library \ + 'icepdf.jar' may not be on the classpath - PDF Pilot disabled."; ### # General Error Messages # Command Line Errors -viewer.commandLin.error=Usage: java org.icepdf.ri.viewer.Main [-loadfile ] [-loadurl ] +viewer.commandLin.error = \ + Usage: java org.icepdf.ri.viewer.Main [-loadfile ] [-loadurl ] # Launcher errors -viewer.launcher.URLError.dialog.title=ICEsoft ICEpdf -viewer.launcher.URLError.dialog.message=ICEpdf could not open the specified file. {0} at URL: {1}. -viewer.launcher.lookAndFeel.error.message=The specified look-and-feel ({0}) is not accessible from this platform. +viewer.launcher.URLError.dialog.title =ICEsoft ICEpdf +viewer.launcher.URLError.dialog.message = ICEpdf could not open the specified file. {0} at URL: {1}. +viewer.launcher.lookAndFeel.error.message = The specified look-and-feel ({0}) is not accessible from this platform. # Pilot Loading Errors ### parser error dialogs -parse.title=Properties Parsing Error -parse.integer=Warning : {0} is not a correct integer. -parse.float=Warning : {0} is not a correct float. -parse.double=Warning : {0} is not a correct double. -parse.choice=Warning : {0} is not a valid choice. -parse.laf=Warning : look-and-feel {0} is not supported. +parse.title = Properties Parsing Error +parse.integer = Warning : {0} is not a correct integer. +parse.float = Warning : {0} is not a correct float. +parse.double = Warning : {0} is not a correct double. +parse.choice = Warning : {0} is not a valid choice. +parse.laf = Warning : look-and-feel {0} is not supported. ### Properties Manager Errors -manager.properties.title=ICEpdf Properties Manager -fontManager.properties.title=ICEpdf Font Manager +manager.properties.title = ICEpdf Properties Manager +fontManager.properties.title = ICEpdf Font Manager -manager.properties.createNewDirectory=To create the directory {0},\nwhere the ICEpdf Viewer will store changes to its setup, click Yes.\n\nIf you click "No", all changes you make to the ICEpdf Viewer setup\nwill be lost when you quit the application. \n\n +manager.properties.createNewDirectory = \ + To create the directory {0},\n\ + where the ICEpdf Viewer will store changes to its setup, click Yes.\n\n\ + If you click "No", all changes you make to the ICEpdf Viewer setup\n\ + will be lost when you quit the application. \n\n -manager.properties.failedCreation=ICEpdf Viewer directory to store user data can not be created:\n{0}\nICEpdf Viewer will not save changes to its default setup. +manager.properties.failedCreation = \ + ICEpdf Viewer directory to store user data can not be created:\n\ + {0}\n\ + ICEpdf Viewer will not save changes to its default setup. -manager.properties.session.nolock=Error creating the lock file :\n{0}\n +manager.properties.session.nolock = \ + Error creating the lock file :\n\ + {0}\n -manager.properties.session.readError=Error loading properties file: \n{0} +manager.properties.session.readError = \ + Error loading properties file: \n\ + {0} -manager.properties.deleted=Property file has been deleted\n({0})\nRecreate it ? +manager.properties.deleted = Property file has been deleted\n\ + ({0})\n\ + Recreate it ? -manager.properties.modified=Property file has been modified since last update\n({0,date,long})\nWould you like to merge changes in the file with the current properties? +manager.properties.modified = Property file has been modified since last update\n\ +({0,date,long})\n\ +Would you like to merge changes in the file with the current properties? -manager.properties.saveError=Impossible to save property file.\nEncountered the folowing error :\n{0} +manager.properties.saveError = Impossible to save property file.\n\ +Encountered the folowing error :\n\ +{0} -manager.properties.lafError=Look&Feel {0} given in the default properties is unsupported.\nUsing system default. +manager.properties.lafError =\ + Look&Feel {0} given in the default properties is unsupported.\n\ + Using system default. -manager.properties.brokenProperty=Broken default property {0} value: {1} +manager.properties.brokenProperty = Broken default property {0} value: {1} -manager.properties.missingProperty=Missing default property {0} value: {1} +manager.properties.missingProperty = Missing default property {0} value: {1} diff --git a/Core/src/org/sleuthkit/autopsy/core/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/core/Bundle.properties-MERGED index 51f1208f61..1d50092e80 100755 --- a/Core/src/org/sleuthkit/autopsy/core/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/core/Bundle.properties-MERGED @@ -3,7 +3,13 @@ Installer.closing.confirmationDialog.title=Ingest is Running # {0} - exception message Installer.closing.messageBox.caseCloseExceptionMessage=Error closing case: {0} OpenIDE-Module-Display-Category=Infrastructure -OpenIDE-Module-Long-Description=This is the core Autopsy module.\n\nThe module contains the core components needed for the bare application to run; the RCP platform, windowing GUI, sleuthkit bindings, datamodel / storage, explorer, result viewers, content viewers, ingest framework, reporting, and core tools, such as the file search.\n\nThe framework included in the module contains APIs for developing modules for ingest, viewers and reporting. The modules can be deployed as Plugins using the Autopsy plugin installer.\nThis module should not be uninstalled - without it, Autopsy will not run.\n\nFor more information, see http://www.sleuthkit.org/autopsy/ +OpenIDE-Module-Long-Description=\ + This is the core Autopsy module.\n\n\ + The module contains the core components needed for the bare application to run; the RCP platform, windowing GUI, sleuthkit bindings, datamodel / storage, explorer, result viewers, content viewers, ingest framework, reporting, and core tools, such as the file search.\n\n\ + The framework included in the module contains APIs for developing modules for ingest, viewers and reporting. \ + The modules can be deployed as Plugins using the Autopsy plugin installer.\n\ + This module should not be uninstalled - without it, Autopsy will not run.\n\n\ + For more information, see http://www.sleuthkit.org/autopsy/ OpenIDE-Module-Name=Autopsy-Core OpenIDE-Module-Short-Description=Autopsy Core Module org_sleuthkit_autopsy_core_update_center=http://sleuthkit.org/autopsy/updates.xml diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED index 609d68bcd1..0636340b0b 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED @@ -75,9 +75,9 @@ DataContentViewerHex.totalPageLabel.text_1=100 DataContentViewerHex.pageLabel2.text=Page # Product Information panel -LBL_Description=

\n Product Version: {0} ({9})
Sleuth Kit Version: {7}
Netbeans RCP Build: {8}
Java: {1}; {2}
System: {3}; {4}; {5}
Userdir: {6}
+LBL_Description=
\n Product Version: {0} ({9})
Sleuth Kit Version: {7}
Netbeans RCP Build: {8}
Java: {1}; {2}
System: {3}; {4}; {5}
Userdir: {6}
Format_OperatingSystem_Value={0} version {1} running on {2} -LBL_Copyright=
Autopsy™ is a digital forensics platform based on The Sleuth Kit™ and other tools.
Copyright © 2003-2020.
+LBL_Copyright=
Autopsy™ is a digital forensics platform based on The Sleuth Kit™ and other tools.
Copyright © 2003-2020.
SortChooser.dialogTitle=Choose Sort Criteria ThumbnailViewChildren.progress.cancelling=(Cancelling) # {0} - file name @@ -105,7 +105,7 @@ DataResultViewerThumbnail.pageNextButton.text= DataResultViewerThumbnail.imagesLabel.text=Images: DataResultViewerThumbnail.imagesRangeLabel.text=- DataResultViewerThumbnail.pageNumLabel.text=- -DataResultViewerThumbnail.filePathLabel.text=\ +DataResultViewerThumbnail.filePathLabel.text=\ \ \ DataResultViewerThumbnail.goToPageLabel.text=Go to Page: DataResultViewerThumbnail.goToPageField.text= AdvancedConfigurationDialog.cancelButton.text=Cancel diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/coreutils/Bundle.properties-MERGED index 18e279dd2c..a0d535f8e6 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/coreutils/Bundle.properties-MERGED @@ -30,7 +30,9 @@ PlatformUtil.getProcVmUsed.sigarNotInit.msg=Cannot get virt mem used, sigar not PlatformUtil.getProcVmUsed.gen.msg=Cannot get virt mem used, {0} PlatformUtil.getJvmMemInfo.usageText=JVM heap usage: {0}, JVM non-heap usage: {1} PlatformUtil.getPhysicalMemInfo.usageText=Physical memory usage (max, total, free): {0}, {1}, {2} -PlatformUtil.getAllMemUsageInfo.usageText={0}\n{1}\nProcess Virtual Memory: {2} +PlatformUtil.getAllMemUsageInfo.usageText={0}\n\ +{1}\n\ +Process Virtual Memory: {2} # {0} - file name ReadImageTask.mesageText=Reading image: {0} StringExtract.illegalStateException.cannotInit.msg=Unicode table not properly initialized, cannot instantiate StringExtract diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED index 16a5f738fb..3aa5e90bd3 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED @@ -45,6 +45,7 @@ AttachmentNode.getActions.openInExtViewer.text=Open in External Viewer Ctrl+E AttachmentNode.getActions.searchFilesSameMD5.text=Search for files with the same MD5 hash AttachmentNode.getActions.viewFileInDir.text=View File in Directory AttachmentNode.getActions.viewInNewWin.text=View in New Window +# {0} - node name BaseChildFactory.NoSuchEventBusException.message=No event bus for node: {0} BlackboardArtifactNode.createSheet.artifactDetails.displayName=Result Details BlackboardArtifactNode.createSheet.artifactDetails.name=Result Details @@ -271,10 +272,10 @@ ImageNode.getActions.viewInNewWin.text=View in New Window ImageNode.createSheet.name.name=Name ImageNode.createSheet.name.displayName=Name ImageNode.createSheet.name.desc=no description -Installer.exception.tskVerStringNull.msg=Sleuth Kit JNI test call returned without error, but version string was null! -Installer.exception.taskVerStringBang.msg=Sleuth Kit JNI test call returned without error, but version string was ""! -Installer.tskLibErr.msg=Problem with Sleuth Kit JNI. Test call failed!\n\nDetails: {0} -Installer.tskLibErr.err=Fatal Error! +Installer.exception.tskVerStringNull.msg=Sleuth Kit JNI test call returned without error, but version string was null\! +Installer.exception.taskVerStringBang.msg=Sleuth Kit JNI test call returned without error, but version string was ""\! +Installer.tskLibErr.msg=Problem with Sleuth Kit JNI. Test call failed\!\n\nDetails: {0} +Installer.tskLibErr.err=Fatal Error\! InterestingHits.interestingItems.text=INTERESTING ITEMS InterestingHits.displayName.text=Interesting Items InterestingHits.createSheet.name.name=Name diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED index 4f0fdfbb56..93c5ed7987 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED @@ -16,7 +16,7 @@ KnownStatusSearchPanel.knownCheckBox.text=Known Status: KnownStatusSearchPanel.knownBadOptionCheckBox.text=Notable KnownStatusSearchPanel.knownOptionCheckBox.text=Known (NSRL or other) KnownStatusSearchPanel.unknownOptionCheckBox.text=Unknown -DateSearchFilter.noneSelectedMsg.text=At least one date type must be selected! +DateSearchFilter.noneSelectedMsg.text=At least one date type must be selected\! DateSearchPanel.dateCheckBox.text=Date: DateSearchPanel.jLabel4.text=Timezone: DateSearchPanel.createdCheckBox.text=Created @@ -57,7 +57,7 @@ FileSearchPanel.search.results.details=Large number of matches may impact perfor FileSearchPanel.search.exception.noFilterSelected.msg=At least one filter must be selected. FileSearchPanel.search.validationErr.msg=Validation Error: {0} FileSearchPanel.emptyWhereClause.text=Invalid options, nothing to show. -KnownStatusSearchFilter.noneSelectedMsg.text=At least one known status must be selected! +KnownStatusSearchFilter.noneSelectedMsg.text=At least one known status must be selected\! NameSearchFilter.emptyNameMsg.text=Must enter something for name search. SizeSearchPanel.sizeCompareComboBox.equalTo=equal to SizeSearchPanel.sizeCompareComboBox.greaterThan=greater than diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED index f852f1c3f3..b6380f2ee5 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED @@ -144,7 +144,7 @@ IngestJob.cancelReason.outOfDiskSpace.text=Out of disk space IngestJob.cancelReason.servicesDown.text=Services Down IngestJob.cancelReason.caseClosed.text=Case closed IngestJobSettingsPanel.globalSettingsButton.text=Global Settings -gest= +gest IngestJobSettingsPanel.globalSettingsButton.actionCommand=Advanced IngestJobSettingsPanel.globalSettingsButton.text=Global Settings IngestJobSettingsPanel.pastJobsButton.text=History diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 9793a57cf1..c5704e873e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -174,7 +174,7 @@ public final class IngestJob { } // Streaming ingest jobs will only have one data source IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); - streamingIngestPipeline.notifyDataSourceReady(); + streamingIngestPipeline.notifyStreamedDataSourceReady(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index c0715c6658..f960aaf575 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -96,25 +96,25 @@ final class IngestJobPipeline { private static enum Stages { /* * The pipeline is instantiating ingest modules and loading them into - * its ingest task pipelines. + * its child ingest module pipelines. */ INITIALIZATION, /* * The pipeline is running file ingest modules on files streamed to it - * by a data source processor and data artifact ingest modules on - * artifacts generated by the analysis of the streamed files. The data - * source has not been added to the pipeline yet. + * by a data source processor. If configured to have data artifact + * ingest modules, it is running them on artifacts generated by the + * analysis of the streamed files. */ - FILE_STREAMING, + FIRST_STAGE_FILE_STREAMING, /* * The pipeline is running the following three types of ingest modules: * higher priority data source level ingest modules, file ingest - * modules, and artifact ingest modules. + * modules, and data artifact ingest modules. */ - FIRST_STAGE, + FIRST_STAGE_ALL_TASKS, /** * The pipeline is running lower priority, usually long-running, data - * source level ingest modules and artifact ingest modules. + * source level ingest modules and data artifact ingest modules. */ SECOND_STAGE, /** @@ -123,7 +123,6 @@ final class IngestJobPipeline { FINALIZATION }; private volatile Stages stage = IngestJobPipeline.Stages.INITIALIZATION; - private final Object stageTransitionLock = new Object(); /* * An ingest pipeline has separate data source level ingest module pipelines @@ -137,11 +136,11 @@ final class IngestJobPipeline { /* * An ingest pipeline has a collection of identical file ingest module * pipelines, one for each file ingest thread in the ingest manager. The - * file ingest threads take file ingest pipeline copies as they need them - * for each file ingest task in the ingest job and return the pipelines - * using a blocking queue. Additionally, a fixed list of all of the file - * ingest module pipelines is used to bypass the blocking queue when cycling - * through the pipelines to make ingest progress snapshots. + * file ingest threads take and return file ingest pipeline copies from a + * blocking queue as they work through the file ingest tasks for the ingest + * job. Additionally, a fixed list of all of the file ingest module + * pipelines is used to bypass the blocking queue when cycling through the + * pipelines to make ingest progress snapshots. */ private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); private final List fileIngestPipelines = new ArrayList<>(); @@ -154,13 +153,13 @@ final class IngestJobPipeline { /* * An ingest pipeline supports cancellation of analysis by individual data * source level ingest modules or cancellation of all remaining analysis by - * ingest modules. Cancellation works by setting flags that are checked by - * the ingest module pipelines every time they transition from one module to - * another. Ingest modules are also expected to check these flags (via the - * ingest job context) and stop processing if they are set. This approach to - * cancellation means that there can be a variable length delay between a - * cancellation request and its fulfillment. Analysis already completed at - * the time that cancellation occurs is not discarded. + * all of its ingest modules. Cancellation works by setting flags that are + * checked by the ingest module pipelines every time they transition from + * one module to another. Ingest modules are also expected to check these + * flags (via the ingest job context) and stop processing if they are set. + * This approach to cancellation means that there can be a variable length + * delay between a cancellation request and its fulfillment. Analysis + * already completed at the time that cancellation occurs is not discarded. */ private volatile boolean currentDataSourceIngestModuleCancelled; private final List cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>(); @@ -169,9 +168,10 @@ final class IngestJobPipeline { /* * An ingest pipeline interacts with the ingest task scheduler to create - * ingest tasks for the content of the data source that is the subject of - * the ingest job and to queue them for the ingest manager's ingest threads. - * Ingest tasks are the units of work for the child ingest module pipelines. + * ingest tasks for analyzing the data source, files and data artifacts that + * are the subject of the ingest job. The scheduler queues the tasks for the + * ingest manager's ingest threads. The ingest tasks are the units of work + * for the ingest pipeline's child ingest module pipelines. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); @@ -193,7 +193,7 @@ final class IngestJobPipeline { /* * Ingest job details are tracked using this object and are recorded in the - * case database. + * case database when the pipeline starts up and shuts down. */ private volatile IngestJobInfo ingestJobInfo; @@ -682,7 +682,7 @@ final class IngestJobPipeline { */ private void startFirstStage() { logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS - stage = Stages.FIRST_STAGE; + stage = Stages.FIRST_STAGE_ALL_TASKS; /* * Do a count of the files the data source processor has added to the @@ -742,7 +742,7 @@ final class IngestJobPipeline { */ private void startFileStreaming() { logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS - stage = Stages.FILE_STREAMING; + stage = Stages.FIRST_STAGE_FILE_STREAMING; if (doUI) { /* @@ -779,21 +779,25 @@ final class IngestJobPipeline { * Notifies the ingest pipeline running in streaming mode that the data * source is now ready for analysis. */ - void notifyDataSourceReady() { + void notifyStreamedDataSourceReady() { logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS - stage = IngestJobPipeline.Stages.FIRST_STAGE; + stage = IngestJobPipeline.Stages.FIRST_STAGE_ALL_TASKS; currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; /* * Do a count of the files the data source processor has added to the * case database. This estimate will be used for ingest progress * snapshots and for the file ingest progress bar if running with a GUI. - * The count will be off by any streamed files that have already been - * analyzed. */ - long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; + long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; synchronized (fileIngestProgressLock) { + if (processedFiles <= filesToProcess) { + filesToProcess -= processedFiles; + } estimatedFilesToProcess = filesToProcess; + if (doUI && fileIngestProgressBar != null) { + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + } } if (doUI) { @@ -808,10 +812,11 @@ final class IngestJobPipeline { } else { /* * If no data source level ingest task is scheduled at this time and - * all of the file level and artifact ingest tasks scheduled when - * streaming began have already executed, there will never be a - * stage completion check in an ingest thread executing an ingest - * task, so such a job would run forever without a check here. + * all of the file level and artifact ingest tasks scheduled during + * the initial file streaming stage have already executed, there + * will never be a stage completion check in an ingest thread + * executing an ingest task, so such a job would run forever without + * a check here. */ checkForStageCompleted(); } @@ -920,12 +925,12 @@ final class IngestJobPipeline { * completed and does a stage transition if they are. */ private void checkForStageCompleted() { - if (stage == Stages.FILE_STREAMING) { + if (stage == Stages.FIRST_STAGE_FILE_STREAMING) { return; } if (taskScheduler.currentTasksAreCompleted(this)) { switch (stage) { - case FIRST_STAGE: + case FIRST_STAGE_ALL_TASKS: finishFirstStage(); break; case SECOND_STAGE: @@ -981,7 +986,7 @@ final class IngestJobPipeline { shutDownIngestModulePipeline(currentDataSourceIngestPipeline); shutDownIngestModulePipeline(artifactIngestPipeline); - if (doUI) { + if (doUI) { synchronized (dataSourceIngestProgressLock) { if (dataSourceIngestProgressBar != null) { dataSourceIngestProgressBar.finish(); @@ -995,15 +1000,15 @@ final class IngestJobPipeline { fileIngestProgressBar = null; } } - + synchronized (artifactIngestProgressLock) { if (artifactIngestProgressBar != null) { artifactIngestProgressBar.finish(); artifactIngestProgressBar = null; } - } + } } - + if (ingestJobInfo != null) { if (cancelled) { try { @@ -1095,10 +1100,6 @@ final class IngestJobPipeline { synchronized (fileIngestProgressLock) { ++processedFiles; if (doUI) { - /** - * Update the file ingest progress bar in the lower - * right hand corner of the main application window. - */ if (processedFiles <= estimatedFilesToProcess) { fileIngestProgressBar.progress(file.getName(), (int) processedFiles); } else { @@ -1172,7 +1173,7 @@ final class IngestJobPipeline { */ void addStreamedFiles(List fileObjIds) { if (hasFileIngestModules()) { - if (stage.equals(Stages.FILE_STREAMING)) { + if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING)) { IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); @@ -1188,8 +1189,8 @@ final class IngestJobPipeline { * @param files A list of the files to add. */ void addFiles(List files) { - if (stage.equals(Stages.FILE_STREAMING) - || stage.equals(Stages.FIRST_STAGE)) { + if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING) + || stage.equals(Stages.FIRST_STAGE_ALL_TASKS)) { taskScheduler.fastTrackFileIngestTasks(this, files); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); @@ -1213,8 +1214,8 @@ final class IngestJobPipeline { */ void addDataArtifacts(List artifacts) { List artifactsToAnalyze = new ArrayList<>(artifacts); - if (stage.equals(Stages.FILE_STREAMING) - || stage.equals(Stages.FIRST_STAGE) + if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING) + || stage.equals(Stages.FIRST_STAGE_ALL_TASKS) || stage.equals(Stages.SECOND_STAGE)) { taskScheduler.scheduleDataArtifactIngestTasks(this, artifactsToAnalyze); } else { diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties-MERGED index e8411caa04..efee783e8f 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties-MERGED @@ -12,7 +12,12 @@ ExtractArchiveWithPasswordAction.progress.text=Unpacking contents of archive: {0 ExtractArchiveWithPasswordAction.prompt.text=Enter Password ExtractArchiveWithPasswordAction.prompt.title=Enter Password OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Embedded File Extraction Ingest Module\n\nThe Embedded File Extraction Ingest Module processes document files (such as doc, docx, ppt, pptx, xls, xlsx) and archive files (such as zip and others archive types supported by the 7zip extractor).\nContents of these files are extracted and the derived files are added back to the current ingest to be processed by the configured ingest modules.\nIf the derived file happens to be an archive file, it will be re-processed by the 7zip extractor - the extractor will process archive files N-levels deep.\n\nThe extracted files are navigable in the directory tree.\n\nThe module is supported on Windows, Linux and Mac operating systems. +OpenIDE-Module-Long-Description=\ + Embedded File Extraction Ingest Module\n\nThe Embedded File Extraction Ingest Module processes document files (such as doc, docx, ppt, pptx, xls, xlsx) and archive files (such as zip and others archive types supported by the 7zip extractor).\n\ + Contents of these files are extracted and the derived files are added back to the current ingest to be processed by the configured ingest modules.\n\ + If the derived file happens to be an archive file, it will be re-processed by the 7zip extractor - the extractor will process archive files N-levels deep.\n\n\ + The extracted files are navigable in the directory tree.\n\n\ + The module is supported on Windows, Linux and Mac operating systems. OpenIDE-Module-Name=Embedded File Extraction OpenIDE-Module-Short-Description=Embedded File Extraction Ingest Module EmbeddedFileExtractorIngestModule.SevenZipContentReadStream.seek.exception.invalidOrigin=Invalid seek origin: {0} diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties-MERGED index 5063bd55fa..cfaadf1635 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties-MERGED @@ -36,27 +36,27 @@ FileExtMismatchSettingsPanel.jLabel1.text=File Types: FileExtMismatchSettingsPanel.newExtButton.text=New Extension FileExtMismatchSettingsPanel.newMimePrompt.message=Add a new MIME file type: FileExtMismatchSettingsPanel.newMimePrompt.title=New MIME -FileExtMismatchSettingsPanel.newMimePrompt.emptyMime.message=MIME type text is empty! +FileExtMismatchSettingsPanel.newMimePrompt.emptyMime.message=MIME type text is empty\! FileExtMismatchSettingsPanel.newMimePrompt.emptyMime.title=Empty type -FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotSupported.message=MIME type not supported! +FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotSupported.message=MIME type not supported\! FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotSupported.title=Type not supported -FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeExists.message=MIME type already exists! +FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeExists.message=MIME type already exists\! FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeExists.title=Type already exists FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotDetectable.message=MIME type is not detectable by this module. FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotDetectable.title=Type not detectable -FileExtMismatchSettingsPanel.removeTypeButton.noneSelected.message=No MIME type selected! +FileExtMismatchSettingsPanel.removeTypeButton.noneSelected.message=No MIME type selected\! FileExtMismatchSettingsPanel.removeTypeButton.noneSelected.title=No type selected FileExtMismatchSettingsPanel.newExtPrompt.message=Add an allowed extension: FileExtMismatchSettingsPanel.newExtPrompt.title=New allowed extension -FileExtMismatchSettingsPanel.newExtPrompt.empty.message=Extension text is empty! +FileExtMismatchSettingsPanel.newExtPrompt.empty.message=Extension text is empty\! FileExtMismatchSettingsPanel.newExtPrompt.empty.title=Extension text empty -FileExtMismatchSettingsPanel.newExtPrompt.noMimeType.message=No MIME type selected! +FileExtMismatchSettingsPanel.newExtPrompt.noMimeType.message=No MIME type selected\! FileExtMismatchSettingsPanel.newExtPrompt.noMimeType.title=No MIME type selected -FileExtMismatchSettingsPanel.newExtPrompt.extExists.message=Extension already exists! +FileExtMismatchSettingsPanel.newExtPrompt.extExists.message=Extension already exists\! FileExtMismatchSettingsPanel.newExtPrompt.extExists.title=Extension already exists -FileExtMismatchSettingsPanel.removeExtButton.noneSelected.message=No extension selected! +FileExtMismatchSettingsPanel.removeExtButton.noneSelected.message=No extension selected\! FileExtMismatchSettingsPanel.removeExtButton.noneSelected.title=No extension selected -FileExtMismatchSettingsPanel.removeExtButton.noMimeTypeSelected.message=No MIME type selected! +FileExtMismatchSettingsPanel.removeExtButton.noMimeTypeSelected.message=No MIME type selected\! FileExtMismatchSettingsPanel.removeExtButton.noMimeTypeSelected.title=No MIME type selected FileExtMismatchSettingsPanel.removeTypeButton.toolTipText= FileExtMismatchModuleSettingsPanel.checkAllRadioButton.text=Check all file types diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties-MERGED index 8dbb55e35f..dd5aa258cc 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties-MERGED @@ -61,7 +61,10 @@ ImportCentralRepoDbProgressDialog.errorParsingFile.message=Error parsing hash se ImportCentralRepoDbProgressDialog.linesProcessed.message=\ hashes processed ImportCentralRepoDbProgressDialog.title.text=Central Repository Import Progress OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Hash Set ingest module. \n\nThe ingest module analyzes files in the disk image and marks them as "known" (based on NSRL hashset lookup for "known" files) and "bad / interesting" (based on one or more hash sets supplied by the user).\n\nThe module also contains additional non-ingest tools that are integrated in the GUI, such as file lookup by hash and hash set configuration. +OpenIDE-Module-Long-Description=\ + Hash Set ingest module. \n\n\ + The ingest module analyzes files in the disk image and marks them as "known" (based on NSRL hashset lookup for "known" files) and "bad / interesting" (based on one or more hash sets supplied by the user).\n\n\ + The module also contains additional non-ingest tools that are integrated in the GUI, such as file lookup by hash and hash set configuration. OpenIDE-Module-Name=HashDatabases OptionsCategory_Name_HashDatabase=Hash Sets OptionsCategory_Keywords_HashDatabase=Hash Sets @@ -188,7 +191,10 @@ HashDbSearchThread.name.searching=Searching HashDbSearchThread.noMoreFilesWithMD5Msg=No other files with the same MD5 hash were found. ModalNoButtons.indexingDbsTitle=Indexing hash sets ModalNoButtons.indexingDbTitle=Indexing hash set -ModalNoButtons.exitHashDbIndexingMsg=You are about to exit out of indexing your hash sets. \nThe generated index will be left unusable. If you choose to continue,\nplease delete the corresponding -md5.idx file in the hash folder.\nExit indexing? +ModalNoButtons.exitHashDbIndexingMsg=You are about to exit out of indexing your hash sets. \n\ +The generated index will be left unusable. If you choose to continue,\n\ + please delete the corresponding -md5.idx file in the hash folder.\n\ + Exit indexing? ModalNoButtons.dlgTitle.unfinishedIndexing=Unfinished Indexing ModalNoButtons.indexThis.currentlyIndexing1Db=Currently indexing 1 hash set ModalNoButtons.indexThese.currentlyIndexing1OfNDbs=Currently indexing 1 of {0} diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties-MERGED index cccbcc1b57..6fb258f014 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties-MERGED @@ -2,6 +2,7 @@ FilesIdentifierIngestJobSettingsPanel.getError=Error getting interesting files s FilesIdentifierIngestJobSettingsPanel.updateError=Error updating interesting files sets settings file. FilesIdentifierIngestModule.getFilesError=Error getting interesting files sets from file. FilesIdentifierIngestModule.indexError.message=Failed to index interesting file hit artifact for keyword search. +# {0} - daysIncluded FilesSet.rule.dateRule.toString=(modified within {0} day(s)) FilesSetDefsPanel.bytes=Bytes FilesSetDefsPanel.cancelImportMsg=Cancel import @@ -121,8 +122,8 @@ FilesSetRulePanel.nameTextField.text= FilesSetRulePanel.ruleNameLabel.text=Rule Name (Optional): FilesSetRulePanel.messages.emptyNameCondition=You must specify a name pattern for this rule. FilesSetRulePanel.messages.invalidNameRegex=The name regular expression is not valid:\n\n{0} -FilesSetRulePanel.messages.invalidCharInName=The name cannot contain \\, /, :, *, ?, ", <, or > unless it is a regular expression. -FilesSetRulePanel.messages.invalidCharInPath=The path cannot contain \\, :, *, ?, ", <, or > unless it is a regular expression. +FilesSetRulePanel.messages.invalidCharInName=The name cannot contain \\, /, :, *, ?, \", <, or > unless it is a regular expression. +FilesSetRulePanel.messages.invalidCharInPath=The path cannot contain \\, :, *, ?, \", <, or > unless it is a regular expression. FilesSetRulePanel.messages.invalidPathRegex=The path regular expression is not valid:\n\n{0} FilesSetDefsPanel.doFileSetsDialog.duplicateRuleSet.text=Rule set with name {0} already exists. FilesSetRulePanel.pathSeparatorInfoLabel.text=Folder must be in parent path. Use '/' to give consecutive names diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties-MERGED index f5dd54dc50..1d07988e4c 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties-MERGED @@ -24,7 +24,7 @@ PhotoRecIngestModule.complete.totalParsetime=Total Parsing Time: PhotoRecIngestModule.complete.photoRecResults=PhotoRec Results PhotoRecIngestModule.NotEnoughDiskSpace.detail.msg=PhotoRec error processing {0} with {1} Not enough space on primary disk to save unallocated space. PhotoRecIngestModule.cancelledByUser=PhotoRec cancelled by user. -PhotoRecIngestModule.error.exitValue=PhotoRec carver returned error exit value = {0} when scanning {1} +PhotoRecIngestModule.error.exitValue=PhotoRec carver returned error exit value \= {0} when scanning {1} PhotoRecIngestModule.error.msg=Error processing {0} with PhotoRec carver. PhotoRecIngestModule.complete.numberOfErrors=Number of Errors while Carving: PhotoRecCarverIngestJobSettingsPanel.detectionSettingsLabel.text=PhotoRec Settings diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/html/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/report/modules/html/Bundle.properties-MERGED index fce93671b3..3db1b822ea 100755 --- a/Core/src/org/sleuthkit/autopsy/report/modules/html/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/report/modules/html/Bundle.properties-MERGED @@ -5,8 +5,8 @@ ReportHTML.getName.text=HTML Report ReportHTML.getDesc.text=A report about results and tagged items in HTML format. ReportHTML.writeIndex.title=for case {0} ReportHTML.writeIndex.noFrames.msg=Your browser is not compatible with our frame setup. -ReportHTML.writeIndex.noFrames.seeNav=Please see the navigation page for artifact links, -ReportHTML.writeIndex.seeSum=and the summary page for a case summary. +ReportHTML.writeIndex.noFrames.seeNav=Please see the navigation page for artifact links, +ReportHTML.writeIndex.seeSum=and the summary page for a case summary. ReportHTML.writeNav.title=Report Navigation ReportHTML.writeNav.h1=Report Navigation ReportHTML.writeNav.summary=Case Summary @@ -16,7 +16,7 @@ ReportHTML.writeSum.caseNumber=Case Number: ReportHTML.writeSum.caseNumImages=Number of data sources in case: ReportHTML.writeSum.examiner=Examiner: ReportHTML.writeSum.title=Case Summary -ReportHTML.writeSum.warningMsg=Warning, this report was run before ingest services completed! +ReportHTML.writeSum.warningMsg=Warning, this report was run before ingest services completed\! # # autopsy/test/scripts/regression.py._html_report_diff() uses reportGenOn.text, caseName, caseNum, # examiner as a regex signature to skip report.html and summary.html From 168cdd4672c3dd784dc8615ee3f429a7ea595e74 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 4 Jun 2021 12:05:29 -0400 Subject: [PATCH 19/63] 7332 artifact pipeline work --- .../autopsy/ingest/IngestJobPipeline.java | 374 ++++++++++-------- 1 file changed, 200 insertions(+), 174 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index f960aaf575..6f6b50bd62 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -31,6 +31,7 @@ import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; +import javax.annotation.concurrent.GuardedBy; import javax.swing.JOptionPane; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; @@ -124,6 +125,13 @@ final class IngestJobPipeline { }; private volatile Stages stage = IngestJobPipeline.Stages.INITIALIZATION; + /* + * The stage field is volatile to allow it to be read by multiple threads. + * This lock is used not to guard the stage field, but to make stage + * transitions atomic. + */ + private final Object stageTransitionLock = new Object(); + /* * An ingest pipeline has separate data source level ingest module pipelines * for the first and second stages. Longer running, lower priority modules @@ -563,9 +571,9 @@ final class IngestJobPipeline { recordIngestJobStartUpInfo(); if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { if (parentJob.getIngestMode() == IngestJob.Mode.STREAMING) { - startFileStreaming(); + startFirstStageInStreamingMode(); } else { - startFirstStage(); + startFirstStageInBatchMode(); } } else if (hasSecondStageDataSourceIngestModules()) { startSecondStage(); @@ -680,58 +688,62 @@ final class IngestJobPipeline { * of the files in the data source (excepting carved and derived files) have * already been added to the case database by the data source processor. */ - private void startFirstStage() { - logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS - stage = Stages.FIRST_STAGE_ALL_TASKS; + private void startFirstStageInBatchMode() { + synchronized (stageTransitionLock) { + logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS + stage = Stages.FIRST_STAGE_ALL_TASKS; - /* - * Do a count of the files the data source processor has added to the - * case database. This estimate will be used for ingest progress - * snapshots and for the file ingest progress bar if running with a GUI. - */ - if (hasFileIngestModules()) { - long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; - synchronized (fileIngestProgressLock) { - estimatedFilesToProcess = filesToProcess; - } - } - - /* - * If running with a GUI, start ingest progress bars in the lower right - * hand corner of the main application window. - */ - if (doUI) { + /* + * Do a count of the files the data source processor has added to + * the case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a + * GUI. + */ if (hasFileIngestModules()) { - startFileIngestProgressBar(); + long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; + synchronized (fileIngestProgressLock) { + estimatedFilesToProcess = filesToProcess; + } } - if (hasFirstStageDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } - if (hasDataArtifactIngestModules()) { - startArtifactIngestProgressBar(); - } - } - /* - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + /* + * If running with a GUI, start ingest progress bars in the lower + * right hand corner of the main application window. + */ + if (doUI) { + if (hasFileIngestModules()) { + startFileIngestProgressBar(); + } + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + if (hasDataArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } + } - /* - * Schedule the first stage ingest tasks and then immediately check for - * stage completion. This is necessary because it is possible that zero - * tasks will actually make it to task execution due to the file filter - * or other ingest job settings. In that case, there will never be a - * stage completion check in an ingest thread executing an ingest task, - * so such a job would run forever without a check here. - */ - if (!files.isEmpty() && hasFileIngestModules()) { - taskScheduler.scheduleFileIngestTasks(this, files); - } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { - taskScheduler.scheduleIngestTasks(this); + /* + * Make the first stage data source level ingest pipeline the + * current data source level pipeline. + */ + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + + /* + * Schedule the first stage ingest tasks and then immediately check + * for stage completion. This is necessary because it is possible + * that zero tasks will actually make it to task execution due to + * the file filter or other ingest job settings. In that case, there + * will never be a stage completion check in an ingest thread + * executing an ingest task, so such a job would run forever without + * a check here. + */ + if (!files.isEmpty() && hasFileIngestModules()) { + taskScheduler.scheduleFileIngestTasks(this, files); + } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { + taskScheduler.scheduleIngestTasks(this); + } + checkForStageCompleted(); } - checkForStageCompleted(); } /** @@ -740,38 +752,40 @@ final class IngestJobPipeline { * adds them to the case database and file level analysis can begin before * data source level analysis. */ - private void startFileStreaming() { - logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS - stage = Stages.FIRST_STAGE_FILE_STREAMING; + private void startFirstStageInStreamingMode() { + synchronized (stageTransitionLock) { + logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS + stage = Stages.FIRST_STAGE_FILE_STREAMING; - if (doUI) { - /* - * If running with a GUI, start ingest progress bars in the lower - * right hand corner of the main application window. - */ - if (hasFileIngestModules()) { + if (doUI) { /* - * Note that because estimated files remaining to process still - * has its initial value of zero, the progress bar will start in - * the "indeterminate" state. An estimate of the files to - * process can be computed in + * If running with a GUI, start ingest progress bars in the + * lower right hand corner of the main application window. */ - startFileIngestProgressBar(); + if (hasFileIngestModules()) { + /* + * Note that because estimated files remaining to process + * still has its initial value of zero, the progress bar + * will start in the "indeterminate" state. An estimate of + * the files to process can be computed in + */ + startFileIngestProgressBar(); + } + if (hasDataArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } } - if (hasDataArtifactIngestModules()) { - startArtifactIngestProgressBar(); - } - } - if (hasDataArtifactIngestModules()) { - /* - * Schedule artifact ingest tasks for any artifacts currently in the - * case database. This needs to be done before any files or the data - * source are streamed in to avoid analyzing data artifacts added to - * the case database by the data source level or file level ingest - * tasks. - */ - taskScheduler.scheduleDataArtifactIngestTasks(this); + if (hasDataArtifactIngestModules()) { + /* + * Schedule artifact ingest tasks for any artifacts currently in + * the case database. This needs to be done before any files or + * the data source are streamed in to avoid analyzing data + * artifacts added to the case database by the data source level + * or file level ingest tasks. + */ + taskScheduler.scheduleDataArtifactIngestTasks(this); + } } } @@ -780,45 +794,48 @@ final class IngestJobPipeline { * source is now ready for analysis. */ void notifyStreamedDataSourceReady() { - logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS - stage = IngestJobPipeline.Stages.FIRST_STAGE_ALL_TASKS; - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + synchronized (stageTransitionLock) { + logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS + stage = IngestJobPipeline.Stages.FIRST_STAGE_ALL_TASKS; + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - /* - * Do a count of the files the data source processor has added to the - * case database. This estimate will be used for ingest progress - * snapshots and for the file ingest progress bar if running with a GUI. - */ - long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; - synchronized (fileIngestProgressLock) { - if (processedFiles <= filesToProcess) { - filesToProcess -= processedFiles; - } - estimatedFilesToProcess = filesToProcess; - if (doUI && fileIngestProgressBar != null) { - fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); - } - } - - if (doUI) { - if (hasFirstStageDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } - } - - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - if (hasFirstStageDataSourceIngestModules()) { - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); - } else { /* - * If no data source level ingest task is scheduled at this time and - * all of the file level and artifact ingest tasks scheduled during - * the initial file streaming stage have already executed, there - * will never be a stage completion check in an ingest thread - * executing an ingest task, so such a job would run forever without - * a check here. + * Do a count of the files the data source processor has added to + * the case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a + * GUI. */ - checkForStageCompleted(); + long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; + synchronized (fileIngestProgressLock) { + if (processedFiles <= filesToProcess) { + filesToProcess -= processedFiles; + } + estimatedFilesToProcess = filesToProcess; + if (doUI && fileIngestProgressBar != null) { + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + } + } + + if (doUI) { + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + } + + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + if (hasFirstStageDataSourceIngestModules()) { + IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + /* + * If no data source level ingest task is scheduled at this time + * and all of the file level and artifact ingest tasks scheduled + * during the initial file streaming stage have already + * executed, there will never be a stage completion check in an + * ingest thread executing an ingest task, so such a job would + * run forever without a check here. + */ + checkForStageCompleted(); + } } } @@ -826,16 +843,18 @@ final class IngestJobPipeline { * Starts the second stage ingest task pipelines. */ private void startSecondStage() { - if (!cancelled && hasSecondStageDataSourceIngestModules()) { - logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS - stage = IngestJobPipeline.Stages.SECOND_STAGE; + synchronized (stageTransitionLock) { + if (!cancelled && hasSecondStageDataSourceIngestModules()) { + logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS + stage = IngestJobPipeline.Stages.SECOND_STAGE; - if (doUI) { - startDataSourceIngestProgressBar(); + if (doUI) { + startDataSourceIngestProgressBar(); + } + + currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; + taskScheduler.scheduleDataSourceIngestTask(this); } - - currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; - taskScheduler.scheduleDataSourceIngestTask(this); } } @@ -925,17 +944,19 @@ final class IngestJobPipeline { * completed and does a stage transition if they are. */ private void checkForStageCompleted() { - if (stage == Stages.FIRST_STAGE_FILE_STREAMING) { - return; - } - if (taskScheduler.currentTasksAreCompleted(this)) { - switch (stage) { - case FIRST_STAGE_ALL_TASKS: - finishFirstStage(); - break; - case SECOND_STAGE: - shutDown(); - break; + synchronized (stageTransitionLock) { + if (stage == Stages.FIRST_STAGE_FILE_STREAMING) { + return; + } + if (taskScheduler.currentTasksAreCompleted(this)) { + switch (stage) { + case FIRST_STAGE_ALL_TASKS: + finishFirstStage(); + break; + case SECOND_STAGE: + shutDown(); + break; + } } } } @@ -980,53 +1001,55 @@ final class IngestJobPipeline { * Shuts down the ingest module pipelines and progress bars for this job. */ private void shutDown() { - logInfoMessage("Finished all tasks"); //NON-NLS - stage = IngestJobPipeline.Stages.FINALIZATION; + synchronized (stageTransitionLock) { + logInfoMessage("Finished all tasks"); //NON-NLS + stage = IngestJobPipeline.Stages.FINALIZATION; - shutDownIngestModulePipeline(currentDataSourceIngestPipeline); - shutDownIngestModulePipeline(artifactIngestPipeline); + shutDownIngestModulePipeline(currentDataSourceIngestPipeline); + shutDownIngestModulePipeline(artifactIngestPipeline); - if (doUI) { - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; + if (doUI) { + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + } + + synchronized (fileIngestProgressLock) { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + } + + synchronized (artifactIngestProgressLock) { + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.finish(); + artifactIngestProgressBar = null; + } } } - synchronized (fileIngestProgressLock) { - if (fileIngestProgressBar != null) { - fileIngestProgressBar.finish(); - fileIngestProgressBar = null; + if (ingestJobInfo != null) { + if (cancelled) { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } + } else { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } } - } - - synchronized (artifactIngestProgressLock) { - if (artifactIngestProgressBar != null) { - artifactIngestProgressBar.finish(); - artifactIngestProgressBar = null; - } - } - } - - if (ingestJobInfo != null) { - if (cancelled) { try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); + ingestJobInfo.setEndDateTime(new Date()); } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } - } else { - try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } - } - try { - ingestJobInfo.setEndDateTime(new Date()); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } } @@ -1172,11 +1195,14 @@ final class IngestJobPipeline { * @param fileObjIds The object IDs of the files. */ void addStreamedFiles(List fileObjIds) { - if (hasFileIngestModules()) { - if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING)) { - IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + synchronized (stageTransitionLock) { + if (hasFileIngestModules()) { + if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING)) { + IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); + } else { + // RJCTODO: Why? + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + } } } } @@ -1188,7 +1214,7 @@ final class IngestJobPipeline { * * @param files A list of the files to add. */ - void addFiles(List files) { + void addFiles(List files) { if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING) || stage.equals(Stages.FIRST_STAGE_ALL_TASKS)) { taskScheduler.fastTrackFileIngestTasks(this, files); From 9944c0c0eb89a36a2558826f78c8812b546237d4 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 4 Jun 2021 12:11:31 -0400 Subject: [PATCH 20/63] Revert "7332 artifact pipeline work" This reverts commit 168cdd4672c3dd784dc8615ee3f429a7ea595e74. --- .../autopsy/ingest/IngestJobPipeline.java | 372 ++++++++---------- 1 file changed, 173 insertions(+), 199 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 6f6b50bd62..f960aaf575 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -31,7 +31,6 @@ import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; -import javax.annotation.concurrent.GuardedBy; import javax.swing.JOptionPane; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; @@ -125,13 +124,6 @@ final class IngestJobPipeline { }; private volatile Stages stage = IngestJobPipeline.Stages.INITIALIZATION; - /* - * The stage field is volatile to allow it to be read by multiple threads. - * This lock is used not to guard the stage field, but to make stage - * transitions atomic. - */ - private final Object stageTransitionLock = new Object(); - /* * An ingest pipeline has separate data source level ingest module pipelines * for the first and second stages. Longer running, lower priority modules @@ -571,9 +563,9 @@ final class IngestJobPipeline { recordIngestJobStartUpInfo(); if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { if (parentJob.getIngestMode() == IngestJob.Mode.STREAMING) { - startFirstStageInStreamingMode(); + startFileStreaming(); } else { - startFirstStageInBatchMode(); + startFirstStage(); } } else if (hasSecondStageDataSourceIngestModules()) { startSecondStage(); @@ -688,62 +680,58 @@ final class IngestJobPipeline { * of the files in the data source (excepting carved and derived files) have * already been added to the case database by the data source processor. */ - private void startFirstStageInBatchMode() { - synchronized (stageTransitionLock) { - logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS - stage = Stages.FIRST_STAGE_ALL_TASKS; + private void startFirstStage() { + logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS + stage = Stages.FIRST_STAGE_ALL_TASKS; - /* - * Do a count of the files the data source processor has added to - * the case database. This estimate will be used for ingest progress - * snapshots and for the file ingest progress bar if running with a - * GUI. - */ - if (hasFileIngestModules()) { - long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; - synchronized (fileIngestProgressLock) { - estimatedFilesToProcess = filesToProcess; - } + /* + * Do a count of the files the data source processor has added to the + * case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a GUI. + */ + if (hasFileIngestModules()) { + long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; + synchronized (fileIngestProgressLock) { + estimatedFilesToProcess = filesToProcess; } - - /* - * If running with a GUI, start ingest progress bars in the lower - * right hand corner of the main application window. - */ - if (doUI) { - if (hasFileIngestModules()) { - startFileIngestProgressBar(); - } - if (hasFirstStageDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } - if (hasDataArtifactIngestModules()) { - startArtifactIngestProgressBar(); - } - } - - /* - * Make the first stage data source level ingest pipeline the - * current data source level pipeline. - */ - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - - /* - * Schedule the first stage ingest tasks and then immediately check - * for stage completion. This is necessary because it is possible - * that zero tasks will actually make it to task execution due to - * the file filter or other ingest job settings. In that case, there - * will never be a stage completion check in an ingest thread - * executing an ingest task, so such a job would run forever without - * a check here. - */ - if (!files.isEmpty() && hasFileIngestModules()) { - taskScheduler.scheduleFileIngestTasks(this, files); - } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { - taskScheduler.scheduleIngestTasks(this); - } - checkForStageCompleted(); } + + /* + * If running with a GUI, start ingest progress bars in the lower right + * hand corner of the main application window. + */ + if (doUI) { + if (hasFileIngestModules()) { + startFileIngestProgressBar(); + } + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + if (hasDataArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } + } + + /* + * Make the first stage data source level ingest pipeline the current + * data source level pipeline. + */ + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + + /* + * Schedule the first stage ingest tasks and then immediately check for + * stage completion. This is necessary because it is possible that zero + * tasks will actually make it to task execution due to the file filter + * or other ingest job settings. In that case, there will never be a + * stage completion check in an ingest thread executing an ingest task, + * so such a job would run forever without a check here. + */ + if (!files.isEmpty() && hasFileIngestModules()) { + taskScheduler.scheduleFileIngestTasks(this, files); + } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { + taskScheduler.scheduleIngestTasks(this); + } + checkForStageCompleted(); } /** @@ -752,41 +740,39 @@ final class IngestJobPipeline { * adds them to the case database and file level analysis can begin before * data source level analysis. */ - private void startFirstStageInStreamingMode() { - synchronized (stageTransitionLock) { - logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS - stage = Stages.FIRST_STAGE_FILE_STREAMING; + private void startFileStreaming() { + logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS + stage = Stages.FIRST_STAGE_FILE_STREAMING; - if (doUI) { + if (doUI) { + /* + * If running with a GUI, start ingest progress bars in the lower + * right hand corner of the main application window. + */ + if (hasFileIngestModules()) { /* - * If running with a GUI, start ingest progress bars in the - * lower right hand corner of the main application window. + * Note that because estimated files remaining to process still + * has its initial value of zero, the progress bar will start in + * the "indeterminate" state. An estimate of the files to + * process can be computed in */ - if (hasFileIngestModules()) { - /* - * Note that because estimated files remaining to process - * still has its initial value of zero, the progress bar - * will start in the "indeterminate" state. An estimate of - * the files to process can be computed in - */ - startFileIngestProgressBar(); - } - if (hasDataArtifactIngestModules()) { - startArtifactIngestProgressBar(); - } + startFileIngestProgressBar(); } - if (hasDataArtifactIngestModules()) { - /* - * Schedule artifact ingest tasks for any artifacts currently in - * the case database. This needs to be done before any files or - * the data source are streamed in to avoid analyzing data - * artifacts added to the case database by the data source level - * or file level ingest tasks. - */ - taskScheduler.scheduleDataArtifactIngestTasks(this); + startArtifactIngestProgressBar(); } } + + if (hasDataArtifactIngestModules()) { + /* + * Schedule artifact ingest tasks for any artifacts currently in the + * case database. This needs to be done before any files or the data + * source are streamed in to avoid analyzing data artifacts added to + * the case database by the data source level or file level ingest + * tasks. + */ + taskScheduler.scheduleDataArtifactIngestTasks(this); + } } /** @@ -794,67 +780,62 @@ final class IngestJobPipeline { * source is now ready for analysis. */ void notifyStreamedDataSourceReady() { - synchronized (stageTransitionLock) { - logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS - stage = IngestJobPipeline.Stages.FIRST_STAGE_ALL_TASKS; - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS + stage = IngestJobPipeline.Stages.FIRST_STAGE_ALL_TASKS; + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - /* - * Do a count of the files the data source processor has added to - * the case database. This estimate will be used for ingest progress - * snapshots and for the file ingest progress bar if running with a - * GUI. - */ - long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; - synchronized (fileIngestProgressLock) { - if (processedFiles <= filesToProcess) { - filesToProcess -= processedFiles; - } - estimatedFilesToProcess = filesToProcess; - if (doUI && fileIngestProgressBar != null) { - fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); - } + /* + * Do a count of the files the data source processor has added to the + * case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a GUI. + */ + long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; + synchronized (fileIngestProgressLock) { + if (processedFiles <= filesToProcess) { + filesToProcess -= processedFiles; } - - if (doUI) { - if (hasFirstStageDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } + estimatedFilesToProcess = filesToProcess; + if (doUI && fileIngestProgressBar != null) { + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); } + } - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + if (doUI) { if (hasFirstStageDataSourceIngestModules()) { - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - /* - * If no data source level ingest task is scheduled at this time - * and all of the file level and artifact ingest tasks scheduled - * during the initial file streaming stage have already - * executed, there will never be a stage completion check in an - * ingest thread executing an ingest task, so such a job would - * run forever without a check here. - */ - checkForStageCompleted(); + startDataSourceIngestProgressBar(); } } + + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + if (hasFirstStageDataSourceIngestModules()) { + IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + /* + * If no data source level ingest task is scheduled at this time and + * all of the file level and artifact ingest tasks scheduled during + * the initial file streaming stage have already executed, there + * will never be a stage completion check in an ingest thread + * executing an ingest task, so such a job would run forever without + * a check here. + */ + checkForStageCompleted(); + } } /** * Starts the second stage ingest task pipelines. */ private void startSecondStage() { - synchronized (stageTransitionLock) { - if (!cancelled && hasSecondStageDataSourceIngestModules()) { - logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS - stage = IngestJobPipeline.Stages.SECOND_STAGE; + if (!cancelled && hasSecondStageDataSourceIngestModules()) { + logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS + stage = IngestJobPipeline.Stages.SECOND_STAGE; - if (doUI) { - startDataSourceIngestProgressBar(); - } - - currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; - taskScheduler.scheduleDataSourceIngestTask(this); + if (doUI) { + startDataSourceIngestProgressBar(); } + + currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; + taskScheduler.scheduleDataSourceIngestTask(this); } } @@ -944,19 +925,17 @@ final class IngestJobPipeline { * completed and does a stage transition if they are. */ private void checkForStageCompleted() { - synchronized (stageTransitionLock) { - if (stage == Stages.FIRST_STAGE_FILE_STREAMING) { - return; - } - if (taskScheduler.currentTasksAreCompleted(this)) { - switch (stage) { - case FIRST_STAGE_ALL_TASKS: - finishFirstStage(); - break; - case SECOND_STAGE: - shutDown(); - break; - } + if (stage == Stages.FIRST_STAGE_FILE_STREAMING) { + return; + } + if (taskScheduler.currentTasksAreCompleted(this)) { + switch (stage) { + case FIRST_STAGE_ALL_TASKS: + finishFirstStage(); + break; + case SECOND_STAGE: + shutDown(); + break; } } } @@ -1001,55 +980,53 @@ final class IngestJobPipeline { * Shuts down the ingest module pipelines and progress bars for this job. */ private void shutDown() { - synchronized (stageTransitionLock) { - logInfoMessage("Finished all tasks"); //NON-NLS - stage = IngestJobPipeline.Stages.FINALIZATION; + logInfoMessage("Finished all tasks"); //NON-NLS + stage = IngestJobPipeline.Stages.FINALIZATION; - shutDownIngestModulePipeline(currentDataSourceIngestPipeline); - shutDownIngestModulePipeline(artifactIngestPipeline); + shutDownIngestModulePipeline(currentDataSourceIngestPipeline); + shutDownIngestModulePipeline(artifactIngestPipeline); - if (doUI) { - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; - } - } - - synchronized (fileIngestProgressLock) { - if (fileIngestProgressBar != null) { - fileIngestProgressBar.finish(); - fileIngestProgressBar = null; - } - } - - synchronized (artifactIngestProgressLock) { - if (artifactIngestProgressBar != null) { - artifactIngestProgressBar.finish(); - artifactIngestProgressBar = null; - } + if (doUI) { + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; } } - if (ingestJobInfo != null) { - if (cancelled) { - try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } - } else { - try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } + synchronized (fileIngestProgressLock) { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; } + } + + synchronized (artifactIngestProgressLock) { + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.finish(); + artifactIngestProgressBar = null; + } + } + } + + if (ingestJobInfo != null) { + if (cancelled) { try { - ingestJobInfo.setEndDateTime(new Date()); + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); } + } else { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } + } + try { + ingestJobInfo.setEndDateTime(new Date()); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } } @@ -1195,14 +1172,11 @@ final class IngestJobPipeline { * @param fileObjIds The object IDs of the files. */ void addStreamedFiles(List fileObjIds) { - synchronized (stageTransitionLock) { - if (hasFileIngestModules()) { - if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING)) { - IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); - } else { - // RJCTODO: Why? - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); - } + if (hasFileIngestModules()) { + if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING)) { + IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } } } @@ -1214,7 +1188,7 @@ final class IngestJobPipeline { * * @param files A list of the files to add. */ - void addFiles(List files) { + void addFiles(List files) { if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING) || stage.equals(Stages.FIRST_STAGE_ALL_TASKS)) { taskScheduler.fastTrackFileIngestTasks(this, files); From dbfecb626b5985273bee2a3b5e1a05dd0ae3059b Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 7 Jun 2021 12:04:35 -0400 Subject: [PATCH 21/63] 7332 artifact pipeline work --- .../sleuthkit/autopsy/ingest/IngestJob.java | 2 +- .../autopsy/ingest/IngestJobPipeline.java | 550 +++++++++--------- 2 files changed, 277 insertions(+), 275 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index c5704e873e..321d38c627 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -174,7 +174,7 @@ public final class IngestJob { } // Streaming ingest jobs will only have one data source IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); - streamingIngestPipeline.notifyStreamedDataSourceReady(); + streamingIngestPipeline.addStreamedDataSource(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index f960aaf575..850ff26294 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -100,18 +100,21 @@ final class IngestJobPipeline { */ INITIALIZATION, /* - * The pipeline is running file ingest modules on files streamed to it - * by a data source processor. If configured to have data artifact - * ingest modules, it is running them on artifacts generated by the - * analysis of the streamed files. + * This stage is unique to a streaming mode ingest job. The pipeline is + * running file ingest modules on files streamed to it via + * addStreamedFiles(). If configured to have data artifact ingest + * modules, the pipeline is also running them on data artifacts + * generated by the analysis of the streamed files. This stage ends when + * the data source is streamed to the pipeline via + * addStreamedDataSource(). */ - FIRST_STAGE_FILE_STREAMING, + FIRST_STAGE_STREAMING, /* * The pipeline is running the following three types of ingest modules: * higher priority data source level ingest modules, file ingest * modules, and data artifact ingest modules. */ - FIRST_STAGE_ALL_TASKS, + FIRST_STAGE, /** * The pipeline is running lower priority, usually long-running, data * source level ingest modules and data artifact ingest modules. @@ -124,6 +127,13 @@ final class IngestJobPipeline { }; private volatile Stages stage = IngestJobPipeline.Stages.INITIALIZATION; + /* + * The stage field is volatile to allow it to be read by multiple threads. + * This lock is used not to guard the stage field, but to make stage + * transitions atomic. + */ + private final Object stageTransitionLock = new Object(); + /* * An ingest pipeline has separate data source level ingest module pipelines * for the first and second stages. Longer running, lower priority modules @@ -260,7 +270,7 @@ final class IngestJobPipeline { * @param jythonModules The ingest module templates for modules implemented * using Jython. */ - private static void sortModuleTemplates(final List sortedModules, final Map javaModules, final Map jythonModules) { + private static void addToIngestPipelineTemplate(final List sortedModules, final Map javaModules, final Map jythonModules) { final List autopsyModules = new ArrayList<>(); final List thirdPartyModules = new ArrayList<>(); Stream.concat(javaModules.entrySet().stream(), jythonModules.entrySet().stream()).forEach((templateEntry) -> { @@ -304,7 +314,7 @@ final class IngestJobPipeline { * @param jythonMapping Mapping for Jython ingest module templates. * @param template The ingest module template. */ - private static void addModuleTemplateToImplLangMap(Map mapping, Map jythonMapping, IngestModuleTemplate template) { + private static void addModuleTemplateToSortingMap(Map mapping, Map jythonMapping, IngestModuleTemplate template) { String className = template.getModuleFactory().getClass().getCanonicalName(); String jythonName = getModuleNameFromJythonClassName(className); if (jythonName != null) { @@ -323,25 +333,14 @@ final class IngestJobPipeline { private void createIngestModulePipelines() throws InterruptedException { /* * Get the enabled ingest module templates from the ingest job settings. - * An ingest module template combines an ingest module factory with job - * level ingest module settings to support the creation of any number of - * fully configured instances of a given ingest module. An ingest module - * factory may be able to create multiple types of ingest modules. */ List enabledTemplates = settings.getEnabledIngestModuleTemplates(); /** * Sort the ingest module templates into buckets based on the module - * types the ingest module factory can create. A template may go into - * more than one bucket. The buckets are actually maps of ingest module - * factory class names to ingest module templates. These maps are used - * to go from an ingest module factory class name read from the pipeline - * configuration file to the corresponding ingest module template. - * - * There are actually two maps for each module type bucket. One map is - * for Java modules and the other one is for Jython modules. The - * templates are separated this way so that Java modules that are not in - * the pipeline config file can be placed before the Jython modules. + * types the template can be used to create. A template may go into more + * than one bucket. Each bucket actually consists of two collections: + * one for Java modules and one for Jython modules. */ Map javaDataSourceModuleTemplates = new LinkedHashMap<>(); Map jythonDataSourceModuleTemplates = new LinkedHashMap<>(); @@ -351,86 +350,79 @@ final class IngestJobPipeline { Map jythonArtifactModuleTemplates = new LinkedHashMap<>(); for (IngestModuleTemplate template : enabledTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - addModuleTemplateToImplLangMap(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); + addModuleTemplateToSortingMap(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); } if (template.isFileIngestModuleTemplate()) { - addModuleTemplateToImplLangMap(javaFileModuleTemplates, jythonFileModuleTemplates, template); + addModuleTemplateToSortingMap(javaFileModuleTemplates, jythonFileModuleTemplates, template); } if (template.isDataArtifactIngestModuleTemplate()) { - addModuleTemplateToImplLangMap(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); + addModuleTemplateToSortingMap(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); } } /** - * Take the module templates that have pipeline configuration file - * entries out of the buckets and put them in lists representing ingest - * task pipelines, in the order prescribed by the file. Note that the - * pipeline configuration file currently only supports specifying data - * source level and file ingest module pipeline layouts. + * Take the module templates that have pipeline configuration entries + * out of the buckets and add them to ingest module pipeline templates + * in the order prescribed by the pipeline configuration. */ IngestPipelinesConfiguration pipelineConfig = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = addConfiguredIngestModuleTemplates(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageOneDataSourceIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = addConfiguredIngestModuleTemplates(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageTwoDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = addConfiguredIngestModuleTemplates(javaFileModuleTemplates, jythonFileModuleTemplates, pipelineConfig.getFileIngestPipelineConfig()); - List artifactModuleTemplates = new ArrayList<>(); + List firstStageDataSourcePipelineTemplate = createIngestPipelineTemplate(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageOneDataSourceIngestPipelineConfig()); + List secondStageDataSourcePipelineTemplate = createIngestPipelineTemplate(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageTwoDataSourceIngestPipelineConfig()); + List filePipelineTemplate = createIngestPipelineTemplate(javaFileModuleTemplates, jythonFileModuleTemplates, pipelineConfig.getFileIngestPipelineConfig()); + List artifactPipelineTemplate = new ArrayList<>(); /** - * Add any module templates remaining in the buckets to the appropriate - * ingest task pipeline. Note that any data source level ingest modules - * that were not listed in the configuration file are added to the first - * stage data source pipeline, Java modules are added before Jython - * modules, and Core Autopsy modules are added before third party - * modules. + * Add any ingest module templates remaining in the buckets to the + * appropriate ingest module pipeline templates. Data source level + * ingest modules templates that were not listed in the pipeline + * configuration are added to the first stage data source pipeline + * template, Java modules are added before Jython modules and Core + * Autopsy modules are added before third party modules. */ - sortModuleTemplates(firstStageDataSourceModuleTemplates, javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates); - sortModuleTemplates(fileIngestModuleTemplates, javaFileModuleTemplates, jythonFileModuleTemplates); - sortModuleTemplates(artifactModuleTemplates, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); + addToIngestPipelineTemplate(firstStageDataSourcePipelineTemplate, javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates); + addToIngestPipelineTemplate(filePipelineTemplate, javaFileModuleTemplates, jythonFileModuleTemplates); + addToIngestPipelineTemplate(artifactPipelineTemplate, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); /** - * Construct the actual ingest task pipelines from the ordered lists. + * Construct the ingest module pipelines from the ingest module pipeline + * templates. */ - firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); - secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); + firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourcePipelineTemplate); + secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourcePipelineTemplate); int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); for (int i = 0; i < numberOfFileIngestThreads; ++i) { - FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates); + FileIngestPipeline pipeline = new FileIngestPipeline(this, filePipelineTemplate); fileIngestPipelinesQueue.put(pipeline); fileIngestPipelines.add(pipeline); } - artifactIngestPipeline = new DataArtifactIngestPipeline(this, artifactModuleTemplates); + artifactIngestPipeline = new DataArtifactIngestPipeline(this, artifactPipelineTemplate); } /** - * Uses an input collection of ingest module templates and a pipeline - * configuration, i.e., an ordered list of ingest module factory class - * names, to create an ordered output list of ingest module templates for an - * ingest task pipeline. The ingest module templates are removed from the - * input collection as they are added to the output collection. + * Creates an ingest module pipeline template that can be used to construct + * an ingest module pipeline. * - * @param javaIngestModuleTemplates A mapping of Java ingest module - * factory class names to ingest module - * templates. - * @param jythonIngestModuleTemplates A mapping of Jython ingest module - * factory proxy class names to ingest - * module templates. - * @param pipelineConfig An ordered list of ingest module - * factory class names representing an - * ingest pipeline, read from the - * pipeline configuration file. + * @param javaIngestModuleTemplates Ingest module templates for ingest + * modules implemented using Java. + * @param jythonIngestModuleTemplates Ingest module templates for ingest + * modules implemented using Jython. + * @param pipelineConfig An ordered list of the ingest modules + * that belong in the ingest pipeline for + * which the template is being created. * - * @return An ordered list of ingest module templates, i.e., an - * uninstantiated pipeline. + * @return An ordered list of ingest module templates, i.e., a template for + * creating ingest module pipelines. */ - private static List addConfiguredIngestModuleTemplates(Map javaIngestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { - List templates = new ArrayList<>(); + private static List createIngestPipelineTemplate(Map javaIngestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { + List pipelineTemplate = new ArrayList<>(); for (String moduleClassName : pipelineConfig) { if (javaIngestModuleTemplates.containsKey(moduleClassName)) { - templates.add(javaIngestModuleTemplates.remove(moduleClassName)); + pipelineTemplate.add(javaIngestModuleTemplates.remove(moduleClassName)); } else if (jythonIngestModuleTemplates.containsKey(moduleClassName)) { - templates.add(jythonIngestModuleTemplates.remove(moduleClassName)); + pipelineTemplate.add(jythonIngestModuleTemplates.remove(moduleClassName)); } } - return templates; + return pipelineTemplate; } /** @@ -533,13 +525,7 @@ final class IngestJobPipeline { * @return True or false. */ boolean hasFileIngestModules() { - if (!fileIngestPipelines.isEmpty()) { - /* - * Note that the file ingest task pipelines are identical. - */ - return !fileIngestPipelines.get(0).isEmpty(); - } - return false; + return (fileIngestPipelines.isEmpty() == false); } /** @@ -563,9 +549,9 @@ final class IngestJobPipeline { recordIngestJobStartUpInfo(); if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { if (parentJob.getIngestMode() == IngestJob.Mode.STREAMING) { - startFileStreaming(); + startFirstStageInStreamingMode(); } else { - startFirstStage(); + startFirstStageInBatchMode(); } } else if (hasSecondStageDataSourceIngestModules()) { startSecondStage(); @@ -680,58 +666,62 @@ final class IngestJobPipeline { * of the files in the data source (excepting carved and derived files) have * already been added to the case database by the data source processor. */ - private void startFirstStage() { - logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS - stage = Stages.FIRST_STAGE_ALL_TASKS; + private void startFirstStageInBatchMode() { + synchronized (stageTransitionLock) { + logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS + stage = Stages.FIRST_STAGE; - /* - * Do a count of the files the data source processor has added to the - * case database. This estimate will be used for ingest progress - * snapshots and for the file ingest progress bar if running with a GUI. - */ - if (hasFileIngestModules()) { - long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; - synchronized (fileIngestProgressLock) { - estimatedFilesToProcess = filesToProcess; - } - } - - /* - * If running with a GUI, start ingest progress bars in the lower right - * hand corner of the main application window. - */ - if (doUI) { + /* + * Do a count of the files the data source processor has added to + * the case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a + * GUI. + */ if (hasFileIngestModules()) { - startFileIngestProgressBar(); + long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; + synchronized (fileIngestProgressLock) { + estimatedFilesToProcess = filesToProcess; + } } - if (hasFirstStageDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } - if (hasDataArtifactIngestModules()) { - startArtifactIngestProgressBar(); - } - } - /* - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + /* + * If running with a GUI, start ingest progress bars in the lower + * right hand corner of the main application window. + */ + if (doUI) { + if (hasFileIngestModules()) { + startFileIngestProgressBar(); + } + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + if (hasDataArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } + } - /* - * Schedule the first stage ingest tasks and then immediately check for - * stage completion. This is necessary because it is possible that zero - * tasks will actually make it to task execution due to the file filter - * or other ingest job settings. In that case, there will never be a - * stage completion check in an ingest thread executing an ingest task, - * so such a job would run forever without a check here. - */ - if (!files.isEmpty() && hasFileIngestModules()) { - taskScheduler.scheduleFileIngestTasks(this, files); - } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { - taskScheduler.scheduleIngestTasks(this); + /* + * Make the first stage data source level ingest pipeline the + * current data source level pipeline. + */ + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + + /* + * Schedule the first stage ingest tasks and then immediately check + * for stage completion. This is necessary because it is possible + * that zero tasks will actually make it to task execution due to + * the file filter or other ingest job settings. In that case, there + * will never be a stage completion check in an ingest thread + * executing an ingest task, so such a job would run forever without + * a check here. + */ + if (!files.isEmpty() && hasFileIngestModules()) { + taskScheduler.scheduleFileIngestTasks(this, files); + } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { + taskScheduler.scheduleIngestTasks(this); + } + checkForStageCompleted(); } - checkForStageCompleted(); } /** @@ -740,38 +730,40 @@ final class IngestJobPipeline { * adds them to the case database and file level analysis can begin before * data source level analysis. */ - private void startFileStreaming() { - logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS - stage = Stages.FIRST_STAGE_FILE_STREAMING; + private void startFirstStageInStreamingMode() { + synchronized (stageTransitionLock) { + logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS + stage = Stages.FIRST_STAGE_STREAMING; - if (doUI) { - /* - * If running with a GUI, start ingest progress bars in the lower - * right hand corner of the main application window. - */ - if (hasFileIngestModules()) { + if (doUI) { /* - * Note that because estimated files remaining to process still - * has its initial value of zero, the progress bar will start in - * the "indeterminate" state. An estimate of the files to - * process can be computed in + * If running with a GUI, start ingest progress bars in the + * lower right hand corner of the main application window. */ - startFileIngestProgressBar(); + if (hasFileIngestModules()) { + /* + * Note that because estimated files remaining to process + * still has its initial value of zero, the progress bar + * will start in the "indeterminate" state. An estimate of + * the files to process can be computed in + */ + startFileIngestProgressBar(); + } + if (hasDataArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } } - if (hasDataArtifactIngestModules()) { - startArtifactIngestProgressBar(); - } - } - if (hasDataArtifactIngestModules()) { - /* - * Schedule artifact ingest tasks for any artifacts currently in the - * case database. This needs to be done before any files or the data - * source are streamed in to avoid analyzing data artifacts added to - * the case database by the data source level or file level ingest - * tasks. - */ - taskScheduler.scheduleDataArtifactIngestTasks(this); + if (hasDataArtifactIngestModules()) { + /* + * Schedule artifact ingest tasks for any artifacts currently in + * the case database. This needs to be done before any files or + * the data source are streamed in to avoid analyzing data + * artifacts added to the case database by the data source level + * or file level ingest tasks. + */ + taskScheduler.scheduleDataArtifactIngestTasks(this); + } } } @@ -779,46 +771,49 @@ final class IngestJobPipeline { * Notifies the ingest pipeline running in streaming mode that the data * source is now ready for analysis. */ - void notifyStreamedDataSourceReady() { - logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS - stage = IngestJobPipeline.Stages.FIRST_STAGE_ALL_TASKS; - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + void addStreamedDataSource() { + synchronized (stageTransitionLock) { + logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS + stage = IngestJobPipeline.Stages.FIRST_STAGE; + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - /* - * Do a count of the files the data source processor has added to the - * case database. This estimate will be used for ingest progress - * snapshots and for the file ingest progress bar if running with a GUI. - */ - long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; - synchronized (fileIngestProgressLock) { - if (processedFiles <= filesToProcess) { - filesToProcess -= processedFiles; - } - estimatedFilesToProcess = filesToProcess; - if (doUI && fileIngestProgressBar != null) { - fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); - } - } - - if (doUI) { - if (hasFirstStageDataSourceIngestModules()) { - startDataSourceIngestProgressBar(); - } - } - - currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - if (hasFirstStageDataSourceIngestModules()) { - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); - } else { /* - * If no data source level ingest task is scheduled at this time and - * all of the file level and artifact ingest tasks scheduled during - * the initial file streaming stage have already executed, there - * will never be a stage completion check in an ingest thread - * executing an ingest task, so such a job would run forever without - * a check here. + * Do a count of the files the data source processor has added to + * the case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a + * GUI. */ - checkForStageCompleted(); + long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; + synchronized (fileIngestProgressLock) { + if (processedFiles <= filesToProcess) { + filesToProcess -= processedFiles; + } + estimatedFilesToProcess = filesToProcess; + if (doUI && fileIngestProgressBar != null) { + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + } + } + + if (doUI) { + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + } + + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + if (hasFirstStageDataSourceIngestModules()) { + IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + /* + * If no data source level ingest task is scheduled at this time + * and all of the file level and artifact ingest tasks scheduled + * during the initial file streaming stage have already + * executed, there will never be a stage completion check in an + * ingest thread executing an ingest task, so such a job would + * run forever without a check here. + */ + checkForStageCompleted(); + } } } @@ -826,16 +821,18 @@ final class IngestJobPipeline { * Starts the second stage ingest task pipelines. */ private void startSecondStage() { - if (!cancelled && hasSecondStageDataSourceIngestModules()) { - logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS - stage = IngestJobPipeline.Stages.SECOND_STAGE; + synchronized (stageTransitionLock) { + if (hasSecondStageDataSourceIngestModules()) { + logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS + stage = IngestJobPipeline.Stages.SECOND_STAGE; - if (doUI) { - startDataSourceIngestProgressBar(); + if (doUI) { + startDataSourceIngestProgressBar(); + } + + currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; + taskScheduler.scheduleDataSourceIngestTask(this); } - - currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; - taskScheduler.scheduleDataSourceIngestTask(this); } } @@ -925,17 +922,19 @@ final class IngestJobPipeline { * completed and does a stage transition if they are. */ private void checkForStageCompleted() { - if (stage == Stages.FIRST_STAGE_FILE_STREAMING) { - return; - } - if (taskScheduler.currentTasksAreCompleted(this)) { - switch (stage) { - case FIRST_STAGE_ALL_TASKS: - finishFirstStage(); - break; - case SECOND_STAGE: - shutDown(); - break; + synchronized (stageTransitionLock) { + if (stage == Stages.FIRST_STAGE_STREAMING) { + return; + } + if (taskScheduler.currentTasksAreCompleted(this)) { + switch (stage) { + case FIRST_STAGE: + finishFirstStage(); + break; + case SECOND_STAGE: + shutDown(); + break; + } } } } @@ -945,88 +944,92 @@ final class IngestJobPipeline { * job and starts the second stage, if appropriate. */ private void finishFirstStage() { - logInfoMessage("Finished first stage analysis"); //NON-NLS + synchronized (stageTransitionLock) { + logInfoMessage("Finished first stage analysis"); //NON-NLS - shutDownIngestModulePipeline(currentDataSourceIngestPipeline); - while (!fileIngestPipelinesQueue.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); - shutDownIngestModulePipeline(pipeline); - } + shutDownIngestModulePipeline(currentDataSourceIngestPipeline); + while (!fileIngestPipelinesQueue.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); + shutDownIngestModulePipeline(pipeline); + } - if (doUI) { - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; + if (doUI) { + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + } + + synchronized (fileIngestProgressLock) { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } } } - synchronized (fileIngestProgressLock) { - if (fileIngestProgressBar != null) { - fileIngestProgressBar.finish(); - fileIngestProgressBar = null; - } + if (!cancelled && hasSecondStageDataSourceIngestModules()) { + startSecondStage(); + } else { + shutDown(); } } - - if (!cancelled && hasSecondStageDataSourceIngestModules()) { - startSecondStage(); - } else { - shutDown(); - } } /** * Shuts down the ingest module pipelines and progress bars for this job. */ private void shutDown() { - logInfoMessage("Finished all tasks"); //NON-NLS - stage = IngestJobPipeline.Stages.FINALIZATION; + synchronized (stageTransitionLock) { + logInfoMessage("Finished all tasks"); //NON-NLS + stage = IngestJobPipeline.Stages.FINALIZATION; - shutDownIngestModulePipeline(currentDataSourceIngestPipeline); - shutDownIngestModulePipeline(artifactIngestPipeline); + shutDownIngestModulePipeline(currentDataSourceIngestPipeline); + shutDownIngestModulePipeline(artifactIngestPipeline); - if (doUI) { - synchronized (dataSourceIngestProgressLock) { - if (dataSourceIngestProgressBar != null) { - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; + if (doUI) { + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + } + + synchronized (fileIngestProgressLock) { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + } + + synchronized (artifactIngestProgressLock) { + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.finish(); + artifactIngestProgressBar = null; + } } } - synchronized (fileIngestProgressLock) { - if (fileIngestProgressBar != null) { - fileIngestProgressBar.finish(); - fileIngestProgressBar = null; + if (ingestJobInfo != null) { + if (cancelled) { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } + } else { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } } - } - - synchronized (artifactIngestProgressLock) { - if (artifactIngestProgressBar != null) { - artifactIngestProgressBar.finish(); - artifactIngestProgressBar = null; - } - } - } - - if (ingestJobInfo != null) { - if (cancelled) { try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); + ingestJobInfo.setEndDateTime(new Date()); } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } - } else { - try { - ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } - } - try { - ingestJobInfo.setEndDateTime(new Date()); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); } } @@ -1173,7 +1176,7 @@ final class IngestJobPipeline { */ void addStreamedFiles(List fileObjIds) { if (hasFileIngestModules()) { - if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING)) { + if (stage.equals(Stages.FIRST_STAGE_STREAMING)) { IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); @@ -1189,8 +1192,8 @@ final class IngestJobPipeline { * @param files A list of the files to add. */ void addFiles(List files) { - if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING) - || stage.equals(Stages.FIRST_STAGE_ALL_TASKS)) { + if (stage.equals(Stages.FIRST_STAGE_STREAMING) + || stage.equals(Stages.FIRST_STAGE)) { taskScheduler.fastTrackFileIngestTasks(this, files); } else { logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); @@ -1214,8 +1217,8 @@ final class IngestJobPipeline { */ void addDataArtifacts(List artifacts) { List artifactsToAnalyze = new ArrayList<>(artifacts); - if (stage.equals(Stages.FIRST_STAGE_FILE_STREAMING) - || stage.equals(Stages.FIRST_STAGE_ALL_TASKS) + if (stage.equals(Stages.FIRST_STAGE_STREAMING) + || stage.equals(Stages.FIRST_STAGE) || stage.equals(Stages.SECOND_STAGE)) { taskScheduler.scheduleDataArtifactIngestTasks(this, artifactsToAnalyze); } else { @@ -1525,7 +1528,6 @@ final class IngestJobPipeline { snapShotTime = new Date().getTime(); } tasksSnapshot = taskScheduler.getTasksSnapshotForJob(pipelineId); - } return new Snapshot(dataSource.getName(), From 00852a932740afd985d28fb8f5c37e9c56c32099 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 9 Jun 2021 17:11:40 -0400 Subject: [PATCH 22/63] 7332 artifact pipeline work --- .../autopsy/ingest/Bundle.properties | 8 ++-- .../autopsy/ingest/Bundle.properties-MERGED | 8 ++-- .../autopsy/ingest/IngestJobPipeline.java | 37 +++++++++++-------- .../ingest/IngestProgressSnapshotPanel.java | 33 ++++++----------- 4 files changed, 42 insertions(+), 44 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index 98959b35b5..cda1ec1503 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -84,13 +84,13 @@ IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.jobID=Job ID IngestJobTableModel.colName.jobID=Job ID IngestJobTableModel.colName.dataSource=Data Source IngestJobTableModel.colName.start=Start -IngestJobTableModel.colName.numProcessed=Num Processed +IngestJobTableModel.colName.numProcessed=Files Processed IngestJobTableModel.colName.filesPerSec=Files/Sec IngestJobTableModel.colName.inProgress=In Progress IngestJobTableModel.colName.filesQueued=Files Queued -IngestJobTableModel.colName.dirQueued=Dir Queued -IngestJobTableModel.colName.rootQueued=Root Queued -IngestJobTableModel.colName.streamingQueued=Streaming Queued +IngestJobTableModel.colName.dirQueued=Dirs Queued +IngestJobTableModel.colName.rootQueued=Roots Queued +IngestJobTableModel.colName.streamingQueued=Streamed Files Queued IngestJobTableModel.colName.dsQueued=DS Queued IngestJobTableModel.colName.artifactsQueued=Artifacts Queued ModuleTableModel.colName.module=Module diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED index b6380f2ee5..9c1a1bfa5c 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED @@ -100,13 +100,13 @@ IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.jobID=Job ID IngestJobTableModel.colName.jobID=Job ID IngestJobTableModel.colName.dataSource=Data Source IngestJobTableModel.colName.start=Start -IngestJobTableModel.colName.numProcessed=Num Processed +IngestJobTableModel.colName.numProcessed=Files Processed IngestJobTableModel.colName.filesPerSec=Files/Sec IngestJobTableModel.colName.inProgress=In Progress IngestJobTableModel.colName.filesQueued=Files Queued -IngestJobTableModel.colName.dirQueued=Dir Queued -IngestJobTableModel.colName.rootQueued=Root Queued -IngestJobTableModel.colName.streamingQueued=Streaming Queued +IngestJobTableModel.colName.dirQueued=Dirs Queued +IngestJobTableModel.colName.rootQueued=Roots Queued +IngestJobTableModel.colName.streamingQueued=Streamed Files Queued IngestJobTableModel.colName.dsQueued=DS Queued IngestJobTableModel.colName.artifactsQueued=Artifacts Queued ModuleTableModel.colName.module=Module diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 850ff26294..4fd449ff47 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -678,7 +678,12 @@ final class IngestJobPipeline { * GUI. */ if (hasFileIngestModules()) { - long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; + long filesToProcess; + if (files.isEmpty()) { + filesToProcess = dataSource.accept(new GetFilesCountVisitor()); + } else { + filesToProcess = files.size(); + } synchronized (fileIngestProgressLock) { estimatedFilesToProcess = filesToProcess; } @@ -777,20 +782,22 @@ final class IngestJobPipeline { stage = IngestJobPipeline.Stages.FIRST_STAGE; currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - /* - * Do a count of the files the data source processor has added to - * the case database. This estimate will be used for ingest progress - * snapshots and for the file ingest progress bar if running with a - * GUI. - */ - long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; - synchronized (fileIngestProgressLock) { - if (processedFiles <= filesToProcess) { - filesToProcess -= processedFiles; - } - estimatedFilesToProcess = filesToProcess; - if (doUI && fileIngestProgressBar != null) { - fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + if (hasFileIngestModules()) { + /* + * Do a count of the files the data source processor has added + * to the case database. This estimate will be used for ingest + * progress snapshots and for the file ingest progress bar if + * running with a GUI. + */ + long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; + synchronized (fileIngestProgressLock) { + if (processedFiles <= filesToProcess) { + filesToProcess -= processedFiles; + } + estimatedFilesToProcess = filesToProcess; + if (doUI && fileIngestProgressBar != null) { + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index eb3f7e614f..b15f5723b5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -167,28 +167,19 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { private static final long serialVersionUID = 1L; - private final String[] columnNames = {NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.jobID"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.dataSource"), + private final String[] columnNames = { + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.jobID"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dataSource"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.start"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.numProcessed"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.filesPerSec"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.inProgress"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.filesQueued"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.dirQueued"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.rootQueued"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.streamingQueued"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.dsQueued"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.artifactsQueued")}; + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.numProcessed"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.filesPerSec"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.inProgress"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.filesQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dirQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.rootQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.streamingQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dsQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.artifactsQueued")}; private List jobSnapshots; From f8563bf9c95e0b08bcd1168ba411d48f666f0359 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 15 Jun 2021 09:47:55 -0400 Subject: [PATCH 23/63] 7332 Fix progress calculation in IngestJobPipeline --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 4fd449ff47..b1cc10f0af 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -789,11 +789,8 @@ final class IngestJobPipeline { * progress snapshots and for the file ingest progress bar if * running with a GUI. */ - long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; + long filesToProcess = dataSource.accept(new GetFilesCountVisitor()); synchronized (fileIngestProgressLock) { - if (processedFiles <= filesToProcess) { - filesToProcess -= processedFiles; - } estimatedFilesToProcess = filesToProcess; if (doUI && fileIngestProgressBar != null) { fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); From 1feee3a795e1b922feaa8282196532faf98dfd98 Mon Sep 17 00:00:00 2001 From: Seb2lyon Date: Thu, 15 Jul 2021 19:26:06 +0200 Subject: [PATCH 24/63] Translate updated pages adHocKeywordSearch, central_repo and content_viewer --- docs/doxygen-user_fr/adHocKeywordSearch.dox | 2 +- docs/doxygen-user_fr/central_repo.dox | 10 +++++++--- docs/doxygen-user_fr/content_viewer.dox | 16 +++++++++++----- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/doxygen-user_fr/adHocKeywordSearch.dox b/docs/doxygen-user_fr/adHocKeywordSearch.dox index 315f903225..e6eab014f2 100644 --- a/docs/doxygen-user_fr/adHocKeywordSearch.dox +++ b/docs/doxygen-user_fr/adHocKeywordSearch.dox @@ -80,7 +80,7 @@ Les résultats apparaitront dans une visionneuse de résultats distincte pour ch \section ad_hoc_kw_lists Keyword Lists (Listes de mots-clés) -En plus d'être sélectionnées lors de l'acquisition, les listes de mots-clés peuvent également être exécutées via le bouton "Keyword Lists". Pour plus d'informations sur la configuration de ces listes de mots clés, consultez la section \ref keywordListsTab de la documentation du module d'acquisition. +En plus d'être sélectionnées lors de l'acquisition, les listes de mots-clés peuvent également être exécutées via le bouton "Keyword Lists". Pour plus d'informations sur la configuration de ces listes de mots clés, consultez la section \ref keyword_keywordListsTab de la documentation du module d'acquisition. Les listes créées à l'aide de la boîte de dialogue "Keyword Search Configuration" peuvent être recherchées manuellement par l'utilisateur en appuyant sur le bouton "Keyword Lists" et en cochant les cases correspondant aux listes à rechercher. La recherche peut être limitée à certaines sources de données uniquement en cochant la case située en bas, puis en mettant en surbrillance les sources de données dans lesquelles effectuer la recherche. Plusieurs sources de données peuvent être sélectionnées en utilisant les touches Maj+clic gauche ou Ctrl+clic gauche. Une fois que tout a été configuré, appuyez sur "Search" pour lancer la recherche. La case à cocher "Save search results" détermine si les résultats de la recherche seront enregistrés dans la base de données du cas. diff --git a/docs/doxygen-user_fr/central_repo.dox b/docs/doxygen-user_fr/central_repo.dox index 3c4d1476b1..d6f985ef4b 100644 --- a/docs/doxygen-user_fr/central_repo.dox +++ b/docs/doxygen-user_fr/central_repo.dox @@ -1,4 +1,4 @@ -/*! \page central_repo_page Référentiel central +/*! \page central_repo_page Référentiel central [TOC] @@ -111,6 +111,10 @@ Descriptions des types de propriétés: - Les propriétés d'ICCID ne sont actuellement créées que par des modules Autopsy personnalisés. - Credit Card - Les propriétés de carte de crédit sont créées par le module \ref keyword_search_page. +- OS Account + - Les propriétés de comptes de système d'exploitation sont créés par le processeur de source de données d'image disque et le module \ref recent_activity_page. +- Installed Programs + - Les propriétés des programmes installés sont créées principalement par le module \ref recent_activity_page. - App-specific Accounts (Facebook, Twitter, etc...) - Ces propriétés proviennent principalement du module \ref android_analyzer_page. @@ -146,7 +150,7 @@ Il existe trois paramètres pour le module d'acquisition "Central Repository":