From bc2c1a20a2b986987980a96eba1c4c653e0b567a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 13 May 2014 15:34:54 -0400 Subject: [PATCH] Add one file ingest pipeline per thread --- .../corecomponents/AutopsyOptionsPanel.form | 60 ++++++------- .../corecomponents/AutopsyOptionsPanel.java | 40 +++++---- .../autopsy/corecomponents/Bundle.properties | 1 + .../autopsy/corecomponents/warning16.png | Bin 0 -> 552 bytes .../sleuthkit/autopsy/ingest/IngestJob.java | 8 +- .../autopsy/ingest/IngestManager.java | 79 +++++++----------- 6 files changed, 93 insertions(+), 95 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/corecomponents/warning16.png diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.form index 6f2a24b8b0..ed068c4df6 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.form @@ -25,6 +25,22 @@ + + + + + + + + + + + + + + + + @@ -35,27 +51,11 @@ + + - - - - - - - - - - - - - - - - - - @@ -83,7 +83,10 @@ - + + + + @@ -102,9 +105,6 @@ - - - @@ -118,9 +118,6 @@ - - - @@ -156,9 +153,6 @@ - - - @@ -198,5 +192,15 @@ + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.java index a39246e34b..c9c9ccf40f 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.java @@ -77,6 +77,7 @@ final class AutopsyOptionsPanel extends javax.swing.JPanel { viewsHideKnownCB = new javax.swing.JCheckBox(); jLabel4 = new javax.swing.JLabel(); numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox(); + jLabel5 = new javax.swing.JLabel(); buttonGroup1.add(useBestViewerRB); useBestViewerRB.setSelected(true); @@ -106,6 +107,9 @@ final class AutopsyOptionsPanel extends javax.swing.JPanel { org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(AutopsyOptionsPanel.class, "AutopsyOptionsPanel.jLabel4.text")); // NOI18N + jLabel5.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/corecomponents/warning16.png"))); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(jLabel5, org.openide.util.NbBundle.getMessage(AutopsyOptionsPanel.class, "AutopsyOptionsPanel.jLabel5.text")); // NOI18N + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -113,6 +117,19 @@ final class AutopsyOptionsPanel extends javax.swing.JPanel { .addGroup(layout.createSequentialGroup() .addContainerGap() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(jLabel3) + .addGroup(layout.createSequentialGroup() + .addGap(10, 10, 10) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(keepCurrentViewerRB) + .addComponent(useBestViewerRB) + .addComponent(dataSourcesHideKnownCB) + .addComponent(viewsHideKnownCB) + .addGroup(layout.createSequentialGroup() + .addComponent(numberOfFileIngestThreadsComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGap(18, 18, 18) + .addComponent(jLabel5, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addContainerGap()))) .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(jLabel2) @@ -120,22 +137,10 @@ final class AutopsyOptionsPanel extends javax.swing.JPanel { .addGap(10, 10, 10) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(useLocalTimeRB) - .addComponent(useGMTTimeRB)))) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) - .addGroup(layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(useGMTTimeRB))) .addComponent(jLabel1) - .addComponent(jLabel3) - .addComponent(jLabel4) - .addGroup(layout.createSequentialGroup() - .addGap(10, 10, 10) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(keepCurrentViewerRB) - .addComponent(useBestViewerRB) - .addComponent(dataSourcesHideKnownCB) - .addComponent(viewsHideKnownCB) - .addComponent(numberOfFileIngestThreadsComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)))) - .addGap(0, 0, Short.MAX_VALUE)))) + .addComponent(jLabel4)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)))) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -160,7 +165,9 @@ final class AutopsyOptionsPanel extends javax.swing.JPanel { .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(jLabel4) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(numberOfFileIngestThreadsComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(numberOfFileIngestThreadsComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(jLabel5))) ); }// //GEN-END:initComponents // Variables declaration - do not modify//GEN-BEGIN:variables @@ -171,6 +178,7 @@ final class AutopsyOptionsPanel extends javax.swing.JPanel { private javax.swing.JLabel jLabel2; private javax.swing.JLabel jLabel3; private javax.swing.JLabel jLabel4; + private javax.swing.JLabel jLabel5; private javax.swing.JRadioButton keepCurrentViewerRB; private javax.swing.JComboBox numberOfFileIngestThreadsComboBox; private javax.swing.JRadioButton useBestViewerRB; diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties index b5cef39ee8..fa5c876206 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties @@ -139,3 +139,4 @@ AutopsyOptionsPanel.keepCurrentViewerRB.toolTipText=For example, stay in Hex vie AutopsyOptionsPanel.keepCurrentViewerRB.text=Stay on the same file viewer AutopsyOptionsPanel.jLabel1.text=When selecting a file: AutopsyOptionsPanel.jLabel2.text=When displaying times: +AutopsyOptionsPanel.jLabel5.text=Restart required to take effect diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/warning16.png b/Core/src/org/sleuthkit/autopsy/corecomponents/warning16.png new file mode 100755 index 0000000000000000000000000000000000000000..f5ba881738ae3072e476f3ddbd7dd34d642f06d6 GIT binary patch literal 552 zcmV+@0@wYCP)A&a_P7S}o zG!s6}jEu~GKp4p&41mvo3NbA|1~KhGFq>2ZZB(et$g5@cY9-1{tFW22MeRe4v~wF$QERIaM<- z{{O*nd`23>@mYBc|Np&XP;x8-%MmglnP1j4N=(g;;pf|paKWFi*D(ky8Zq$8n1Dhw z4W9v-%1)IGf4<*f`2Xh#iNdNA(_lT4EX==BZIP28N-#eIhh~>reEIz qPU=w%q=Bs errors = new ArrayList<>(); - int maxNumberOfPipelines = IngestManager.getMaxNumberOfDataSourceIngestThreads(); - for (int i = 0; i < maxNumberOfPipelines; ++i) { + int numberOfPipelines = IngestManager.getInstance().getNumberOfDataSourceIngestThreads(); + for (int i = 0; i < numberOfPipelines; ++i) { DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(context, ingestModuleTemplates); errors.addAll(pipeline.startUp()); dataSourceIngestPipelines.put(pipeline); @@ -84,8 +84,8 @@ final class IngestJob { } } - maxNumberOfPipelines = IngestManager.getMaxNumberOfFileIngestThreads(); - for (int i = 0; i < maxNumberOfPipelines; ++i) { + numberOfPipelines = IngestManager.getInstance().getNumberOfFileIngestThreads(); + for (int i = 0; i < numberOfPipelines; ++i) { FileIngestPipeline pipeline = new FileIngestPipeline(context, ingestModuleTemplates); errors.addAll(pipeline.startUp()); fileIngestPipelines.put(pipeline); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 92964a584d..60fd6c542f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -46,7 +46,6 @@ import org.sleuthkit.autopsy.core.UserPreferences; */ public class IngestManager { - private static final int MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS = 1; private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; @@ -65,6 +64,7 @@ public class IngestManager { private final ConcurrentHashMap> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ConcurrentHashMap> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private volatile IngestMessageTopComponent ingestMessageBox; + private int numberOfFileIngestThreads; /** * Gets the ingest manager. @@ -81,7 +81,7 @@ public class IngestManager { */ private IngestManager() { startDataSourceIngestThread(); - int numberOfFileIngestThreads = UserPreferences.numberOfFileIngestThreads(); + numberOfFileIngestThreads = UserPreferences.numberOfFileIngestThreads(); if ((numberOfFileIngestThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfFileIngestThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) { numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; UserPreferences.setNumberOfFileIngestThreads(numberOfFileIngestThreads); @@ -89,15 +89,6 @@ public class IngestManager { for (int i = 0; i < numberOfFileIngestThreads; ++i) { startFileIngestThread(); } - - UserPreferences.addChangeListener(new PreferenceChangeListener() { - @Override - public void preferenceChange(PreferenceChangeEvent evt) { - if (evt.getKey().equals(UserPreferences.NUMBER_OF_FILE_INGEST_THREADS)) { - setNumberOfFileIngestThreads(); - } - } - }); } /** @@ -112,49 +103,19 @@ public class IngestManager { } /** - * Gets the maximum number of data source ingest threads the ingest manager - * will use. + * Gets the number of data source ingest threads the ingest manager will + * use. */ - public static int getMaxNumberOfDataSourceIngestThreads() { - return MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS; + public int getNumberOfDataSourceIngestThreads() { + return 1; } /** * Gets the maximum number of file ingest threads the ingest manager will * use. */ - public static int getMaxNumberOfFileIngestThreads() { - return MAX_NUMBER_OF_FILE_INGEST_THREADS; - } - - /** - * Changes the number of file ingest threads the ingest manager will use to - * no more than MAX_NUMBER_OF_FILE_INGEST_THREADS and no less than - * MIN_NUMBER_OF_FILE_INGEST_THREADS. Out of range requests are converted to - * requests for DEFAULT_NUMBER_OF_FILE_INGEST_THREADS. - * - * @param numberOfThreads The desired number of file ingest threads. - */ - public synchronized static void setNumberOfFileIngestThreads() { - int numberOfThreads = UserPreferences.numberOfFileIngestThreads(); - if ((numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) { - numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; - UserPreferences.setNumberOfFileIngestThreads(numberOfThreads); - } - if (instance.fileIngestThreads.size() != numberOfThreads) { - if (instance.fileIngestThreads.size() > numberOfThreads) { - Long[] threadIds = instance.fileIngestThreads.keySet().toArray(new Long[instance.fileIngestThreads.size()]); - int numberOfThreadsToCancel = instance.fileIngestThreads.size() - numberOfThreads; - for (int i = 0; i < numberOfThreadsToCancel; ++i) { - instance.cancelFileIngestThread(threadIds[i]); - } - } else if (instance.fileIngestThreads.size() < numberOfThreads) { - int numberOfThreadsToAdd = numberOfThreads - instance.fileIngestThreads.size(); - for (int i = 0; i < numberOfThreadsToAdd; ++i) { - instance.startFileIngestThread(); - } - } - } + public int getNumberOfFileIngestThreads() { + return numberOfFileIngestThreads; } /** @@ -319,6 +280,30 @@ public class IngestManager { ingestModuleEventPublisher.removePropertyChangeListener(listener); } + /** + * Add an ingest job and ingest module event property change listener. + * + * @deprecated Use addIngestJobEventListener() and/or + * addIngestModuleEventListener(). + * @param listener The PropertyChangeListener to register. + */ + public static void addPropertyChangeListener(final PropertyChangeListener listener) { + instance.ingestJobEventPublisher.addPropertyChangeListener(listener); + instance.ingestModuleEventPublisher.addPropertyChangeListener(listener); + } + + /** + * Remove an ingest job and ingest module event property change listener. + * + * @deprecated Use removeIngestJobEventListener() and/or + * removeIngestModuleEventListener(). + * @param listener The PropertyChangeListener to unregister. + */ + public static void removePropertyChangeListener(final PropertyChangeListener listener) { + instance.ingestJobEventPublisher.removePropertyChangeListener(listener); + instance.ingestModuleEventPublisher.removePropertyChangeListener(listener); + } + /** * Fire an ingest event signifying an ingest job started. *