From 6071a57a7bcad5db610ad4b645aca4459c9de7cc Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 16 Nov 2016 14:24:15 -0500 Subject: [PATCH] Merging --- .../autoingest/AutoIngestDashboard.java | 83 +++-- .../autoingest/AutoIngestManager.java | 295 ++++++++++++------ .../AutoIngestSettingsPanel.form | 13 + .../AutoIngestSettingsPanel.java | 33 +- .../AutoIngestUserPreferences.java | 234 +++++++++++++- .../configuration/Bundle.properties | 2 + .../configuration/SharedConfiguration.java | 5 + 7 files changed, 541 insertions(+), 124 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java index f0d19bdef3..356cd07705 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java @@ -107,6 +107,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer { private AutoIngestManager manager; private ExecutorService updateExecutor; private boolean isPaused; + private boolean autoIngestStarted; private Color pendingTableBackground; private Color pendingTablelForeground; @@ -569,6 +570,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer { boolean enabled = row >= 0 && row < completedTable.getRowCount(); bnDeleteCase.setEnabled(enabled); bnShowCaseLog.setEnabled(enabled); + bnReprocessJob.setEnabled(enabled); }); } @@ -576,16 +578,17 @@ public final class AutoIngestDashboard extends JPanel implements Observer { * Sets the initial state of the buttons on the panel. */ private void initButtons() { - bnOptions.setEnabled(false); + bnOptions.setEnabled(true); bnDeleteCase.setEnabled(false); enablePendingTableButtons(false); bnShowCaseLog.setEnabled(false); - bnPause.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.text")); - bnPause.setToolTipText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.toolTipText")); - bnPause.setEnabled(false); - bnRefresh.setEnabled(false); + bnReprocessJob.setEnabled(false); + bnPause.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnStart.text")); + bnPause.setToolTipText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnStart.toolTipText")); + bnPause.setEnabled(true); //initial label for bnPause is 'Start' and it's enabled for user to start the process + bnRefresh.setEnabled(false); //at initial stage, nothing to refresh enableRunningTableButtons(false); - tbStatusMessage.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.running")); + tbStatusMessage.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnStart.startMessage")); } /** @@ -621,6 +624,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer { */ try { manager.startUp(); + autoIngestStarted = true; } catch (AutoIngestManager.AutoIngestManagerStartupException ex) { SYS_LOGGER.log(Level.SEVERE, "Dashboard error starting up auto ingest", ex); tbStatusMessage.setText(NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.AutoIngestStartupError")); @@ -657,8 +661,13 @@ public final class AutoIngestDashboard extends JPanel implements Observer { updateExecutor.submit(new UpdateAllJobsTablesTask()); manager.scanInputDirsNow(); - bnPause.setEnabled(true); + //bnPause.setEnabled(true); + bnPause.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.text")); + bnPause.setToolTipText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.toolTipText")); bnRefresh.setEnabled(true); + bnOptions.setEnabled(false); + + tbStatusMessage.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.running")); } /** @@ -745,7 +754,10 @@ public final class AutoIngestDashboard extends JPanel implements Observer { "AutoIngestDashboard.PauseDueToSharedConfigError=Paused, unable to update shared configuration.", "AutoIngestDashboard.PauseDueToIngestJobStartFailure=Paused, unable to start ingest job processing.", "AutoIngestDashboard.PauseDueToFileExporterError=Paused, unable to load File Exporter settings.", - "AutoIngestDashboard.bnPause.running=Running" + "AutoIngestDashboard.bnPause.running=Running", + "AutoIngestDashboard.bnStart.startMessage=Waiting to start", + "AutoIngestDashboard.bnStart.text=Start", + "AutoIngestDashboard.bnStart.toolTipText=Start processing auto ingest jobs" }) @Override public void update(Observable o, Object arg) { @@ -762,6 +774,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer { EventQueue.invokeLater(() -> { tbStatusMessage.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.paused")); bnOptions.setEnabled(true); + bnRefresh.setEnabled(false); isPaused = true; }); break; @@ -769,6 +782,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer { EventQueue.invokeLater(() -> { tbStatusMessage.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.PauseDueToSystemError")); bnOptions.setEnabled(true); + bnRefresh.setEnabled(false); pause(false); isPaused = true; setServicesStatusMessage(); @@ -823,6 +837,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer { * currently running job, if any. */ manager.pause(); + bnRefresh.setEnabled(false); } } @@ -842,6 +857,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer { bnPause.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.text")); bnPause.setToolTipText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.toolTipText")); tbStatusMessage.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.running")); + bnRefresh.setEnabled(true); /** * Remove the graying out of the pending table. @@ -1092,6 +1108,16 @@ public final class AutoIngestDashboard extends JPanel implements Observer { SYS_LOGGER.log(Level.SEVERE, "Dashboard error refreshing table", ex); } } + + /** + * Get the current lists of jobs and update the UI. + */ + private void refreshTables(){ + JobsSnapshot jobsSnapshot = manager.getCurrentJobsSnapshot(); + refreshTable(jobsSnapshot.getCompletedJobs(), completedTableModel, null); + refreshTable(jobsSnapshot.getPendingJobs(), pendingTableModel, null); + refreshTable(jobsSnapshot.getRunningJobs(), runningTableModel, null); + } /** * This method is called from within the constructor to initialize the form. @@ -1438,8 +1464,10 @@ public final class AutoIngestDashboard extends JPanel implements Observer { * @param evt - The button click event. */ private void bnRefreshActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_bnRefreshActionPerformed - manager.scanInputDirsNow(); - updateExecutor.submit(new UpdateAllJobsTablesTask()); + this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + manager.scanInputDirsAndWait(); + refreshTables(); + this.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); }//GEN-LAST:event_bnRefreshActionPerformed /** @@ -1476,6 +1504,7 @@ public final class AutoIngestDashboard extends JPanel implements Observer { completedTable.clearSelection(); this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); CaseDeletionResult result = manager.deleteCase(caseName, caseDirectoryPath); + refreshTables(); this.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); if (CaseDeletionResult.FAILED == result) { JOptionPane.showMessageDialog(this, @@ -1488,7 +1517,6 @@ public final class AutoIngestDashboard extends JPanel implements Observer { org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.DeletionFailed"), JOptionPane.INFORMATION_MESSAGE); } - updateExecutor.submit(new UpdateAllJobsTablesTask()); } } }//GEN-LAST:event_bnDeleteCaseActionPerformed @@ -1517,8 +1545,8 @@ public final class AutoIngestDashboard extends JPanel implements Observer { * see it). */ this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - List runningJobs = manager.cancelCurrentJob(); - refreshTable(runningJobs, runningTableModel, null); + manager.cancelCurrentJob(); + refreshTables(); this.setCursor(Cursor.getDefaultCursor()); } }//GEN-LAST:event_bnCancelJobActionPerformed @@ -1540,6 +1568,17 @@ public final class AutoIngestDashboard extends JPanel implements Observer { * @param evt The button click event. */ private void bnPauseActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_bnPauseActionPerformed + + if (!autoIngestStarted) { + //put up a wait cursor during the start up operation + this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + + startUp(); + + this.setCursor(null); + //done for startup + return; + } if (!isPaused) { tbStatusMessage.setText(org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.bnPause.pausing")); pause(true); @@ -1584,8 +1623,8 @@ public final class AutoIngestDashboard extends JPanel implements Observer { * see it). */ this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - List runningJobs = manager.cancelCurrentDataSourceLevelIngestModule(); - refreshTable(runningJobs, runningTableModel, null); + manager.cancelCurrentDataSourceLevelIngestModule(); + refreshTables(); this.setCursor(Cursor.getDefaultCursor()); } }//GEN-LAST:event_bnCancelModuleActionPerformed @@ -1609,8 +1648,8 @@ public final class AutoIngestDashboard extends JPanel implements Observer { if (pendingTableModel.getRowCount() > 0 && pendingTable.getSelectedRow() >= 0) { this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); String caseName = (pendingTableModel.getValueAt(pendingTable.getSelectedRow(), JobsTableModelColumns.CASE.ordinal())).toString(); - List prioritizedQueue = manager.prioritizeCase(caseName); - refreshTable(prioritizedQueue, pendingTableModel, null); + manager.prioritizeCase(caseName); + refreshTables(); pendingTable.clearSelection(); enablePendingTableButtons(false); AutoIngestDashboard.this.setCursor(Cursor.getDefaultCursor()); @@ -1658,8 +1697,8 @@ public final class AutoIngestDashboard extends JPanel implements Observer { if (pendingTableModel.getRowCount() > 0 && pendingTable.getSelectedRow() >= 0) { this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); Path manifestFilePath = (Path) (pendingTableModel.getValueAt(pendingTable.getSelectedRow(), JobsTableModelColumns.MANIFEST_FILE_PATH.ordinal())); - List prioritizedQueue = manager.prioritizeJob(manifestFilePath); - refreshTable(prioritizedQueue, pendingTableModel, null); + manager.prioritizeJob(manifestFilePath); + refreshTables(); pendingTable.clearSelection(); enablePendingTableButtons(false); AutoIngestDashboard.this.setCursor(Cursor.getDefaultCursor()); @@ -1684,10 +1723,8 @@ public final class AutoIngestDashboard extends JPanel implements Observer { } this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); Path manifestPath = (Path) completedTableModel.getValueAt(completedTable.getSelectedRow(), JobsTableModelColumns.MANIFEST_FILE_PATH.ordinal()); - JobsSnapshot jobsSnapshot = manager.reprocessJob(manifestPath); - refreshTable(jobsSnapshot.getCompletedJobs(), completedTableModel, null); - refreshTable(jobsSnapshot.getPendingJobs(), pendingTableModel, null); - refreshTable(jobsSnapshot.getRunningJobs(), runningTableModel, null); + manager.reprocessJob(manifestPath); + refreshTables(); AutoIngestDashboard.this.setCursor(Cursor.getDefaultCursor()); }//GEN-LAST:event_bnReprocessJobActionPerformed diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index aa760e4e18..9af7052735 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -37,6 +37,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import org.sleuthkit.autopsy.modules.vmextractor.VirtualMachineFinder; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.datamodel.CaseDbConnectionInfo; import java.time.Duration; @@ -67,16 +68,31 @@ import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.ThreadSafe; +import javax.swing.filechooser.FileFilter; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import org.apache.commons.io.FilenameUtils; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.CaseActionException; import org.sleuthkit.autopsy.ingest.IngestManager; +import org.openide.modules.InstalledFileLocator; import org.sleuthkit.autopsy.casemodule.Case.CaseType; +import org.sleuthkit.autopsy.casemodule.GeneralFilter; +import org.sleuthkit.autopsy.casemodule.ImageDSProcessor; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.core.ServicesMonitor; import org.sleuthkit.autopsy.core.UserPreferencesException; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; +import org.sleuthkit.autopsy.coreutils.ExecUtil; import org.sleuthkit.autopsy.coreutils.NetworkUtils; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.events.AutopsyEventPublisher; import org.sleuthkit.autopsy.ingest.IngestJob; @@ -89,9 +105,12 @@ import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.CaseMetadata; +import org.sleuthkit.autopsy.casemodule.LocalFilesDSProcessor; import org.sleuthkit.autopsy.core.ServicesMonitor.ServicesMonitorException; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult; +import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.events.AutopsyEventException; +import org.sleuthkit.autopsy.ingest.IngestJob.CancellationReason; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestModuleError; import org.sleuthkit.autopsy.experimental.autoingest.FileExporter.FileExportException; @@ -127,7 +146,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang private static final int NUM_INPUT_SCAN_SCHEDULING_THREADS = 1; private static final String INPUT_SCAN_SCHEDULER_THREAD_NAME = "AIM-input-scan-scheduler-%d"; - private static final long INPUT_SCAN_INTERVAL = 5; // 5 minutes private static final String INPUT_SCAN_THREAD_NAME = "AIM-input-scan-%d"; private static int DEFAULT_JOB_PRIORITY = 0; private static final String AUTO_INGEST_THREAD_NAME = "AIM-job-processing-%d"; @@ -141,8 +159,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang private static final long JOB_STATUS_EVENT_INTERVAL_SECONDS = 10; private static final String JOB_STATUS_PUBLISHING_THREAD_NAME = "AIM-job-status-event-publisher-%d"; private static final long MAX_MISSED_JOB_STATUS_UPDATES = 10; - private static final String TSK_IS_DRIVE_IMAGE_TOOL_DIR = "tsk_isImageTool"; - private static final String TSK_IS_DRIVE_IMAGE_TOOL_EXE = "tsk_isImageTool.exe"; private static final java.util.logging.Logger SYS_LOGGER = AutoIngestSystemLogger.getLogger(); private static AutoIngestManager instance; private final AutopsyEventPublisher eventPublisher; @@ -165,7 +181,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang private CoordinationService coordinationService; private JobProcessingTask jobProcessingTask; private Future jobProcessingTaskFuture; - private Path tskIsImageToolExePath; private Path rootInputDirectory; private Path rootOutputDirectory; private volatile State state; @@ -228,7 +243,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang } rootInputDirectory = Paths.get(AutoIngestUserPreferences.getAutoModeImageFolder()); rootOutputDirectory = Paths.get(AutoIngestUserPreferences.getAutoModeResultsFolder()); - inputScanSchedulingExecutor.scheduleAtFixedRate(new InputDirScanSchedulingTask(), 0, INPUT_SCAN_INTERVAL, TimeUnit.MINUTES); + inputScanSchedulingExecutor.scheduleAtFixedRate(new InputDirScanSchedulingTask(), 0, AutoIngestUserPreferences.getMinutesOfInputScanInterval(), TimeUnit.MINUTES); jobProcessingTask = new JobProcessingTask(); jobProcessingTaskFuture = jobProcessingExecutor.submit(jobProcessingTask); jobStatusPublishingExecutor.scheduleAtFixedRate(new PeriodicJobStatusEventTask(), JOB_STATUS_EVENT_INTERVAL_SECONDS, JOB_STATUS_EVENT_INTERVAL_SECONDS, TimeUnit.SECONDS); @@ -344,7 +359,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang completedJobs.add(event.getJob()); } } - scanInputDirsNow(); + //scanInputDirsNow(); setChanged(); notifyObservers(Event.JOB_COMPLETED); } @@ -486,6 +501,19 @@ public final class AutoIngestManager extends Observable implements PropertyChang } inputScanExecutor.submit(new InputDirScanTask()); } + + /** + * Start a scan of the input directories and wait for scan to complete. + */ + void scanInputDirsAndWait(){ + if (State.RUNNING != state) { + return; + } + SYS_LOGGER.log(Level.INFO, "Starting input scan of {0}", rootInputDirectory); + InputDirScanner scanner = new InputDirScanner(); + scanner.scan(); + SYS_LOGGER.log(Level.INFO, "Completed input scan of {0}", rootInputDirectory); + } /** * Pauses processing of the pending jobs queue. The currently running job @@ -512,16 +540,13 @@ public final class AutoIngestManager extends Observable implements PropertyChang * Bumps the priority of all pending ingest jobs for a specified case. * * @param caseName The name of the case to be prioritized. - * - * @return A snapshot of the pending jobs queue after prioritization. */ - List prioritizeCase(final String caseName) { + void prioritizeCase(final String caseName) { if (state != State.RUNNING) { - return Collections.emptyList(); + return; } - List pendingJobsSnapshot = new ArrayList<>(); List prioritizedJobs = new ArrayList<>(); int maxPriority = 0; synchronized (jobsLock) { @@ -551,7 +576,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang } Collections.sort(pendingJobs, new AutoIngestJob.PriorityComparator()); - pendingJobsSnapshot.addAll(pendingJobs); } if (!prioritizedJobs.isEmpty()) { @@ -559,23 +583,18 @@ public final class AutoIngestManager extends Observable implements PropertyChang eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName)); }).start(); } - - return pendingJobsSnapshot; } /** * Bumps the priority of an auto ingest job. * * @param manifestPath The manifest file path for the job to be prioritized. - * - * @return A snapshot of the pending jobs queue after prioritization. */ - List prioritizeJob(Path manifestPath) { + void prioritizeJob(Path manifestPath) { if (state != State.RUNNING) { - return Collections.emptyList(); + return; } - List pendingJobsSnapshot = new ArrayList<>(); int maxPriority = 0; AutoIngestJob prioritizedJob = null; synchronized (jobsLock) { @@ -603,7 +622,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang } Collections.sort(pendingJobs, new AutoIngestJob.PriorityComparator()); - pendingJobsSnapshot.addAll(pendingJobs); } if (null != prioritizedJob) { @@ -612,8 +630,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName)); }).start(); } - - return pendingJobsSnapshot; } /** @@ -621,9 +637,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang * * @param manifestPath The manifiest file path for the completed job. * - * @return An updated jobs snapshot, */ - JobsSnapshot reprocessJob(Path manifestPath) { + void reprocessJob(Path manifestPath) { AutoIngestJob completedJob = null; synchronized (jobsLock) { for (Iterator iterator = completedJobs.iterator(); iterator.hasNext();) { @@ -650,9 +665,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang } Collections.sort(pendingJobs, new AutoIngestJob.PriorityComparator()); - List runningJobs = new ArrayList<>(); - getJobs(null, runningJobs, null); - return new JobsSnapshot(pendingJobs, runningJobs, completedJobs); } } @@ -810,6 +822,18 @@ public final class AutoIngestManager extends Observable implements PropertyChang } } } + + /** + * Get the current snapshot of the job lists. + * @return Snapshot of jobs lists + */ + JobsSnapshot getCurrentJobsSnapshot(){ + synchronized(jobsLock){ + List runningJobs = new ArrayList<>(); + getJobs(null, runningJobs, null); + return new JobsSnapshot(pendingJobs, runningJobs, completedJobs); + } + } /** * Tries to unload the Solr core for a case. @@ -869,35 +893,30 @@ public final class AutoIngestManager extends Observable implements PropertyChang /** * Starts the process of cancelling the current job. * - * @return The cancelled job plus any jobs running on other nodes. The - * current job is included in the list because it can take some time - * for the automated ingest process for the job to be shut down in - * an orderly fashion. + * Note that the current job is included in the running list for a while + * because it can take some time + * for the automated ingest process for the job to be shut down in + * an orderly fashion. */ - List cancelCurrentJob() { + void cancelCurrentJob() { if (State.RUNNING != state) { - return Collections.emptyList(); + return; } synchronized (jobsLock) { if (null != currentJob) { currentJob.cancel(); SYS_LOGGER.log(Level.INFO, "Cancelling automated ingest for manifest {0}", currentJob.getManifest().getFilePath()); } - List runningJobs = new ArrayList<>(); - getJobs(null, runningJobs, null); - return runningJobs; } } /** * Cancels the currently running data-source-level ingest module for the * current job. - * - * @return The current job plus any jobs running on other nodes. */ - List cancelCurrentDataSourceLevelIngestModule() { + void cancelCurrentDataSourceLevelIngestModule() { if (State.RUNNING != state) { - return Collections.emptyList(); + return; } synchronized (jobsLock) { if (null != currentJob) { @@ -911,10 +930,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang } } } - List runningJobs = new ArrayList<>(); - getJobs(null, runningJobs, null); - return runningJobs; - } } @@ -951,7 +966,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang /** * Scans the input directory tree and refreshes the pending jobs queue - * and the completed jobs list. Crashed job recovery is perfomed as + * and the completed jobs list. Crashed job recovery is performed as * needed. */ @Override @@ -959,14 +974,12 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (Thread.currentThread().isInterrupted()) { return null; } - synchronized (jobsLock) { - SYS_LOGGER.log(Level.INFO, "Starting input scan of {0}", rootInputDirectory); - InputDirScanner scanner = new InputDirScanner(); - scanner.scan(); - SYS_LOGGER.log(Level.INFO, "Completed input scan of {0}", rootInputDirectory); - setChanged(); - notifyObservers(Event.INPUT_SCAN_COMPLETED); - } + SYS_LOGGER.log(Level.INFO, "Starting input scan of {0}", rootInputDirectory); + InputDirScanner scanner = new InputDirScanner(); + scanner.scan(); + SYS_LOGGER.log(Level.INFO, "Completed input scan of {0}", rootInputDirectory); + setChanged(); + notifyObservers(Event.INPUT_SCAN_COMPLETED); return null; } @@ -1389,7 +1402,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang } try { processJobs(); - } catch (Exception ex) { + } catch (Exception ex) { // Exception firewall if (jobProcessingTaskFuture.isCancelled()) { break; } @@ -1598,10 +1611,10 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.INFO, "Started processing pending jobs queue"); Lock manifestLock = JobProcessingTask.this.dequeueAndLockNextJob(); while (null != manifestLock) { - if (currentJob.isCancelled() || jobProcessingTaskFuture.isCancelled()) { - return; - } try { + if (currentJob.isCancelled() || jobProcessingTaskFuture.isCancelled()) { + return; + } processJob(); } finally { manifestLock.release(); @@ -1641,8 +1654,10 @@ public final class AutoIngestManager extends Observable implements PropertyChang * @throws CoordinationServiceException if there is an error while * acquiring or releasing a * manifest file lock. + * @throws InterruptedException if the thread is interrupted while + * reading the lock data */ - private Lock dequeueAndLockNextJob() throws CoordinationServiceException { + private Lock dequeueAndLockNextJob() throws CoordinationServiceException, InterruptedException { SYS_LOGGER.log(Level.INFO, "Checking pending jobs queue for ready job, enforcing max jobs per case"); Lock manifestLock; synchronized (jobsLock) { @@ -1678,8 +1693,10 @@ public final class AutoIngestManager extends Observable implements PropertyChang * @throws CoordinationServiceException if there is an error while * acquiring or releasing a * manifest file lock. + * @throws InterruptedException if the thread is interrupted while + * reading the lock data */ - private Lock dequeueAndLockNextJob(boolean enforceMaxJobsPerCase) throws CoordinationServiceException { + private Lock dequeueAndLockNextJob(boolean enforceMaxJobsPerCase) throws CoordinationServiceException, InterruptedException { Lock manifestLock = null; synchronized (jobsLock) { Iterator iterator = pendingJobs.iterator(); @@ -1696,6 +1713,18 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ continue; } + + ManifestNodeData nodeData = new ManifestNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString())); + if(! nodeData.getStatus().equals(PENDING)){ + /* + * Due to a timing issue or a missed event, + * a non-pending job has ended up on the pending queue. + * Skip the job and remove it from the queue. + */ + iterator.remove(); + continue; + } + if (enforceMaxJobsPerCase) { int currentJobsForCase = 0; for (AutoIngestJob runningJob : hostNamesToRunningJobs.values()) { @@ -1758,6 +1787,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang private void processJob() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutomatedIngestDataSourceProcessor.AutomatedIngestDataSourceProcessorException { Manifest manifest = currentJob.getManifest(); String manifestPath = manifest.getFilePath().toString(); + ManifestNodeData nodeData = new ManifestNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath)); + nodeData.setStatus(PROCESSING); + coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath, nodeData.toArray()); SYS_LOGGER.log(Level.INFO, "Started processing of {0}", manifestPath); currentJob.setStage(AutoIngestJob.Stage.STARTING); setChanged(); @@ -1773,6 +1805,21 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (jobProcessingTaskFuture.isCancelled()) { currentJob.cancel(); } + + nodeData = new ManifestNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath)); + if(currentJob.isCompleted() || currentJob.isCancelled()){ + nodeData.setStatus(COMPLETED); + Date completedDate = new Date(); + currentJob.setCompletedDate(completedDate); + nodeData.setCompletedDate(currentJob.getCompletedDate()); + nodeData.setErrorsOccurred(currentJob.hasErrors()); + } else { + // The job may get retried + nodeData.setStatus(PENDING); + } + coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath, nodeData.toArray()); + + boolean retry = (!currentJob.isCancelled() && !currentJob.isCompleted()); SYS_LOGGER.log(Level.INFO, "Completed processing of {0}, retry = {1}", new Object[]{manifestPath, retry}); if (currentJob.isCancelled()) { @@ -2001,9 +2048,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang private void runIngestForJob(Case caseForJob) throws CoordinationServiceException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutomatedIngestDataSourceProcessor.AutomatedIngestDataSourceProcessorException { Manifest manifest = currentJob.getManifest(); String manifestPath = manifest.getFilePath().toString(); - ManifestNodeData nodeData = new ManifestNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath)); - nodeData.setStatus(PROCESSING); - coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath, nodeData.toArray()); try { if (currentJob.isCancelled() || jobProcessingTaskFuture.isCancelled()) { return; @@ -2012,13 +2056,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang } finally { currentJob.setCompleted(); - Date completedDate = new Date(); - currentJob.setCompletedDate(completedDate); - nodeData = new ManifestNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath)); - nodeData.setStatus(COMPLETED); - nodeData.setCompletedDate(completedDate); - nodeData.setErrorsOccurred(currentJob.hasErrors()); - coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath, nodeData.toArray()); } } @@ -2053,7 +2090,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang return; } - DataSource dataSource = identifyDataSource(); + DataSource dataSource = identifyDataSource(caseForJob); if (null == dataSource) { currentJob.setStage(AutoIngestJob.Stage.COMPLETED); return; @@ -2096,14 +2133,17 @@ public final class AutoIngestManager extends Observable implements PropertyChang * * @return A data source object. * - * @throws IOException if there was an error extracting or - * reading the data source. - * @throws InterruptedException if the thread running the auto ingest - * job processing task is interrupted while - * blocked, i.e., if auto ingest is - * shutting down. + * @throws AutoIngestAlertFileException if there is an error creating an + * alert file. + * @throws AutoIngestJobLoggerException if there is an error writing to + * the auto ingest log for the + * case. + * @throws InterruptedException if the thread running the auto + * ingest job processing task is + * interrupted while blocked, i.e., + * if auto ingest is shutting down. */ - private DataSource identifyDataSource() throws InterruptedException, AutoIngestAlertFileException, AutoIngestJobLoggerException { + private DataSource identifyDataSource(Case caseForJob) throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); SYS_LOGGER.log(Level.INFO, "Identifying data source for {0} ", manifestPath); @@ -2129,11 +2169,15 @@ public final class AutoIngestManager extends Observable implements PropertyChang * * @param dataSource The data source. * - * @throws SystemErrorException if there is an error adding the data - * source. - * @throws InterruptedException if the thread running the job processing - * task is interrupted while blocked, i.e., - * if auto ingest is shutting down. + * @throws AutoIngestAlertFileException if there is an error creating an + * alert file. + * @throws AutoIngestJobLoggerException if there is an error writing to + * the auto ingest log for the + * case. + * @throws InterruptedException if the thread running the job + * processing task is interrupted + * while blocked, i.e., if auto + * ingest is shutting down. */ private void runDataSourceProcessor(Case caseForJob, DataSource dataSource) throws InterruptedException, AutoIngestAlertFileException, AutoIngestJobLoggerException, AutomatedIngestDataSourceProcessor.AutomatedIngestDataSourceProcessorException { Manifest manifest = currentJob.getManifest(); @@ -2408,7 +2452,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang try { FileExporter fileExporter = new FileExporter(); if (fileExporter.isEnabled()) { - fileExporter.process(manifest.getDeviceId(), dataSource.getContent()); + fileExporter.process(manifest.getDeviceId(), dataSource.getContent(), currentJob::isCancelled); jobLogger.logFileExportCompleted(); } else { SYS_LOGGER.log(Level.WARNING, "Exporting files not enabled for {0}", manifestPath); @@ -2659,6 +2703,27 @@ public final class AutoIngestManager extends Observable implements PropertyChang notifyObservers(Event.JOB_STATUS_UPDATED); eventPublisher.publishRemotely(new AutoIngestJobStatusEvent(currentJob)); } + + if(AutoIngestUserPreferences.getStatusDatabaseLoggingEnabled()){ + String message; + boolean isError = false; + if(getErrorState().equals(ErrorState.NONE)){ + if(currentJob != null){ + message = "Processing " + currentJob.getManifest().getDataSourceFileName() + + " for case " + currentJob.getManifest().getCaseName(); + } else { + message = "Paused or waiting for next case"; + } + } else { + message = getErrorState().toString(); + isError = true; + } + try{ + StatusDatabaseLogger.logToStatusDatabase(message, isError); + } catch (SQLException | UserPreferencesException ex){ + SYS_LOGGER.log(Level.WARNING, "Failed to update status database", ex); + } + } } // check whether any remote nodes have timed out @@ -2728,22 +2793,33 @@ public final class AutoIngestManager extends Observable implements PropertyChang } /** - * RJCTODO + * The current auto ingest error state. */ private enum ErrorState { - NONE, - COORDINATION_SERVICE_ERROR, - SHARED_CONFIGURATION_DOWNLOAD_ERROR, - SERVICES_MONITOR_COMMUNICATION_ERROR, - DATABASE_SERVER_ERROR, - KEYWORD_SEARCH_SERVER_ERROR, - CASE_MANAGEMENT_ERROR, - ANALYSIS_STARTUP_ERROR, - FILE_EXPORT_ERROR, - ALERT_FILE_ERROR, - JOB_LOGGER_ERROR, - DATA_SOURCE_PROCESSOR_ERROR, - UNEXPECTED_EXCEPTION + NONE ("None"), + COORDINATION_SERVICE_ERROR ("Coordination service error"), + SHARED_CONFIGURATION_DOWNLOAD_ERROR("Shared configuration download error"), + SERVICES_MONITOR_COMMUNICATION_ERROR ("Services monitor communication error"), + DATABASE_SERVER_ERROR ("Database server error"), + KEYWORD_SEARCH_SERVER_ERROR ("Keyword search server error"), + CASE_MANAGEMENT_ERROR ("Case management error"), + ANALYSIS_STARTUP_ERROR ("Analysis startup error"), + FILE_EXPORT_ERROR ("File export error"), + ALERT_FILE_ERROR ("Alert file error"), + JOB_LOGGER_ERROR ("Job logger error"), + DATA_SOURCE_PROCESSOR_ERROR ("Data source processor error"), + UNEXPECTED_EXCEPTION ("Unknown error"); + + private final String desc; + + private ErrorState(String desc){ + this.desc = desc; + } + + @Override + public String toString(){ + return desc; + } } /** @@ -2808,6 +2884,31 @@ public final class AutoIngestManager extends Observable implements PropertyChang FULLY_DELETED } +// Is this still needed ?????? + /* + private static final class FileFilters { + + private static final List cellebriteLogicalReportFilters = CellebriteXMLProcessor.getFileFilterList(); + private static final List cellebritePhysicalReportFilters = CellebriteAndroidImageProcessor.getFileFilterList(); + private static final GeneralFilter archiveFilter = new GeneralFilter(Arrays.asList(ArchiveUtil.getSupportedArchiveTypes()), ""); + private static final List archiveFilters = new ArrayList<>(); + static { + archiveFilters.add(archiveFilter); + } + + private static boolean isAcceptedByFilter(File file, List filters) { + for (FileFilter filter : filters) { + if (filter.accept(file)) { + return true; + } + } + return false; + } + + private FileFilters() { + } + }*/ + @ThreadSafe private static final class DataSource { diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.form b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.form index 423a85254f..af04281441 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.form +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.form @@ -536,6 +536,8 @@ + + @@ -548,6 +550,7 @@ + @@ -588,6 +591,16 @@ + + + + + + + + + + diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.java index 8025291343..5cdf086aaf 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.java @@ -596,6 +596,7 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { jPanelIngestSettings.setEnabled(mode == OptionsUiMode.AIM); bnEditIngestSettings.setEnabled(mode == OptionsUiMode.AIM); bnAdvancedSettings.setEnabled(mode == OptionsUiMode.AIM); + bnLogging.setEnabled(mode == OptionsUiMode.AIM); jPanelSharedConfig.setEnabled(mode == OptionsUiMode.AIM); sharedConfigCheckbox.setEnabled(mode == OptionsUiMode.AIM); masterNodeCheckBox.setEnabled(mode == OptionsUiMode.AIM && sharedConfigCheckbox.isSelected()); @@ -665,6 +666,7 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { bnEditIngestSettings = new javax.swing.JButton(); bnAdvancedSettings = new javax.swing.JButton(); bnFileExport = new javax.swing.JButton(); + bnLogging = new javax.swing.JButton(); cbJoinAutoIngestCluster = new javax.swing.JCheckBox(); tbOops = new javax.swing.JTextField(); @@ -937,6 +939,13 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { } }); + org.openide.awt.Mnemonics.setLocalizedText(bnLogging, org.openide.util.NbBundle.getMessage(AutoIngestSettingsPanel.class, "AutoIngestSettingsPanel.bnLogging.text")); // NOI18N + bnLogging.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + bnLoggingActionPerformed(evt); + } + }); + javax.swing.GroupLayout jPanelIngestSettingsLayout = new javax.swing.GroupLayout(jPanelIngestSettings); jPanelIngestSettings.setLayout(jPanelIngestSettingsLayout); jPanelIngestSettingsLayout.setHorizontalGroup( @@ -948,6 +957,8 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { .addComponent(bnAdvancedSettings, javax.swing.GroupLayout.PREFERRED_SIZE, 155, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(bnFileExport, javax.swing.GroupLayout.PREFERRED_SIZE, 155, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(bnLogging, javax.swing.GroupLayout.PREFERRED_SIZE, 155, javax.swing.GroupLayout.PREFERRED_SIZE) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); jPanelIngestSettingsLayout.setVerticalGroup( @@ -957,7 +968,8 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { .addGroup(jPanelIngestSettingsLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(bnEditIngestSettings) .addComponent(bnFileExport) - .addComponent(bnAdvancedSettings)) + .addComponent(bnAdvancedSettings) + .addComponent(bnLogging)) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); @@ -1228,6 +1240,23 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { // TODO add your handling code here: }//GEN-LAST:event_tbOopsActionPerformed + private void bnLoggingActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_bnLoggingActionPerformed + JDialog jDialog = new JDialog(); + NodeStatusLogPanel loggingPanel = new NodeStatusLogPanel(jDialog); + + JScrollPane jScrollPane = new JScrollPane(loggingPanel); + jScrollPane.setVerticalScrollBarPolicy(javax.swing.ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED); + jScrollPane.setHorizontalScrollBarPolicy(javax.swing.ScrollPaneConstants.HORIZONTAL_SCROLLBAR_AS_NEEDED); + jScrollPane.setMinimumSize(new Dimension(100, 100)); + jDialog.add(jScrollPane); + jDialog.setTitle(NbBundle.getMessage(AutoIngestSettingsPanel.class, "AutoIngestSettingsPanel.NodeStatusLogging.text")); + jDialog.setIconImage(ImageUtilities.loadImage("org/sleuthkit/autopsy/experimental/images/frame32.gif")); + jDialog.setModalityType(java.awt.Dialog.ModalityType.APPLICATION_MODAL); + jDialog.pack(); + jDialog.setLocationRelativeTo(this); + jDialog.setVisible(true); + }//GEN-LAST:event_bnLoggingActionPerformed + private void enableUI(boolean state) { enableOptionsBasedOnMode(OptionsUiMode.DOWNLOADING_CONFIGURATION); downloadButton.setEnabled(state); @@ -1339,6 +1368,7 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { bnAdvancedSettings.setEnabled(enabled); bnEditIngestSettings.setEnabled(enabled); bnFileExport.setEnabled(enabled); + bnLogging.setEnabled(enabled); browseInputFolderButton.setEnabled(enabled); browseOutputFolderButton.setEnabled(enabled); browseSharedSettingsButton.setEnabled(sharedConfigCheckbox.isSelected() && jRadioButtonAutomated.isSelected()); @@ -1362,6 +1392,7 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { private javax.swing.JButton bnAdvancedSettings; private javax.swing.JButton bnEditIngestSettings; private javax.swing.JButton bnFileExport; + private javax.swing.JButton bnLogging; private javax.swing.JButton browseInputFolderButton; private javax.swing.JButton browseOutputFolderButton; private javax.swing.JButton browseSharedSettingsButton; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestUserPreferences.java b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestUserPreferences.java index 8063873d88..8f866b8c59 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestUserPreferences.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestUserPreferences.java @@ -18,10 +18,18 @@ */ package org.sleuthkit.autopsy.experimental.configuration; +import java.util.Base64; import java.util.prefs.PreferenceChangeListener; import java.util.prefs.Preferences; import java.util.prefs.BackingStoreException; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.PBEParameterSpec; +import org.openide.util.NbBundle; import org.openide.util.NbPreferences; +import org.sleuthkit.autopsy.core.UserPreferencesException; /** * Provides convenient access to a Preferences node for auto ingest user preferences @@ -47,8 +55,16 @@ public final class AutoIngestUserPreferences { private static final String SHARED_CONFIG_MASTER = "SharedSettingsMaster"; // NON-NLS private static final String AUTO_MODE_CONTEXT_STRING = "AutoModeContext"; // NON-NLS private static final String SLEEP_BETWEEN_CASES_TIME = "SleepBetweenCasesTime"; // NON-NLS + private static final String SHOW_TOOLS_WARNING = "ShowToolsWarning"; // NON-NLS private static final String MAX_NUM_TIMES_TO_PROCESS_IMAGE = "MaxNumTimesToAttemptToProcessImage"; // NON-NLS private static final String MAX_CONCURRENT_NODES_FOR_ONE_CASE = "MaxConcurrentNodesForOneCase"; // NON-NLS + private static final String STATUS_DATABASE_LOGGING_ENABLED = "StatusDatabaseLoggingEnabled"; // NON-NLS + private static final String LOGGING_DB_HOSTNAME_OR_IP = "LoggingHostnameOrIP"; // NON-NLS + private static final String LOGGING_PORT = "LoggingPort"; // NON-NLS + private static final String LOGGING_USERNAME = "LoggingUsername"; // NON-NLS + private static final String LOGGING_PASSWORD = "LoggingPassword"; // NON-NLS + private static final String LOGGING_DATABASE_NAME = "LoggingDatabaseName"; // NON-NLS + private static final String INPUT_SCAN_INTERVAL_TIME = "IntervalBetweenInputScan"; // NON-NLS // Prevent instantiation. private AutoIngestUserPreferences() { @@ -234,6 +250,24 @@ public final class AutoIngestUserPreferences { return AUTO_MODE_CONTEXT_STRING; } + /** + * Save whether tools warning dialog should be shown on startup. + * + * @param showToolsWarning true = show warning dialog, false = don't show + */ + public static void setShowToolsWarning(boolean showToolsWarning) { + preferences.putBoolean(SHOW_TOOLS_WARNING, showToolsWarning); + } + + /** + * Retrieve tools warning dialog setting. + * + * @return + */ + public static boolean getShowToolsWarning() { + return preferences.getBoolean(SHOW_TOOLS_WARNING, true); + } + /** * Get the configured time to sleep between cases to prevent * database locks @@ -246,8 +280,10 @@ public final class AutoIngestUserPreferences { } /** - * Set the configured time to sleep between cases to prevent - * database locks + * Sets the wait time used by auto ingest nodes to ensure proper + * synchronization of node operations in circumstances where delays may + * occur, e.g., network file system latency effects on the visibility of + * newly created shared directories and files. * * @param int value the number of seconds to sleep between cases */ @@ -296,5 +332,197 @@ public final class AutoIngestUserPreferences { */ public static void setMaxConcurrentIngestNodesForOneCase(int numberOfNodes) { preferences.putInt(MAX_CONCURRENT_NODES_FOR_ONE_CASE, numberOfNodes); - } + } + + /** + * Get status database logging checkbox state for automated ingest mode from + * persistent storage. + * + * @return Boolean true if database logging is enabled. + */ + public static Boolean getStatusDatabaseLoggingEnabled() { + return preferences.getBoolean(STATUS_DATABASE_LOGGING_ENABLED, false); + } + + /** + * Save status database logging checkbox state for automated ingest mode to + * persistent storage. + * + * @param databaseLoggingEnabled true = use database logging in auto-ingest + * mode + */ + public static void setStatusDatabaseLoggingEnabled(boolean databaseLoggingEnabled) { + preferences.putBoolean(STATUS_DATABASE_LOGGING_ENABLED, databaseLoggingEnabled); + } + + /** + * Get the logging database hostname from persistent storage. + * + * @return Logging database hostname or IP + */ + public static String getLoggingDatabaseHostnameOrIP() { + return preferences.get(LOGGING_DB_HOSTNAME_OR_IP, ""); + } + + /** + * Save the logging database hostname to persistent storage. + * + * @param hostname Logging database hostname or IP + */ + public static void setLoggingDatabaseHostnameOrIP(String hostname) { + preferences.put(LOGGING_DB_HOSTNAME_OR_IP, hostname); + } + + /** + * Get the logging database port from persistent storage. + * + * @return logging database port + */ + public static String getLoggingPort() { + return preferences.get(LOGGING_PORT, ""); + } + + /** + * Save the logging database port to persistent storage. + * + * @param port Logging database port + */ + public static void setLoggingPort(String port) { + preferences.put(LOGGING_PORT, port); + } + + /** + * Get the logging database username from persistent storage. + * + * @return logging database username + */ + public static String getLoggingUsername() { + return preferences.get(LOGGING_USERNAME, ""); + } + + /** + * Save the logging database username to persistent storage. + * + * @param username Logging database username + */ + public static void setLoggingUsername(String username) { + preferences.put(LOGGING_USERNAME, username); + } + + /** + * Get the logging database password from persistent storage. + * + * @return logging database password + */ + public static String getLoggingPassword() throws UserPreferencesException { + return TextConverter.convertHexTextToText(preferences.get(LOGGING_PASSWORD, "")); + } + + /** + * Save the logging database password to persistent storage. + * + * @param password Logging database password + */ + public static void setLoggingPassword(String password) throws UserPreferencesException { + preferences.put(LOGGING_PASSWORD, TextConverter.convertTextToHexText(password)); + } + + /** + * Get the logging database name from persistent storage. + * + * @return logging database name + */ + public static String getLoggingDatabaseName() { + return preferences.get(LOGGING_DATABASE_NAME, ""); + } + + /** + * Save the logging database name to persistent storage. + * + * @param name Logging database name + */ + public static void setLoggingDatabaseName(String name) { + preferences.put(LOGGING_DATABASE_NAME, name); + } + + /** + * Get the configured time for input scan interval + * + * @return int the value in minutes, default is 60 minutes. + */ + public static int getMinutesOfInputScanInterval() { + int answer = Integer.parseInt(preferences.get(INPUT_SCAN_INTERVAL_TIME, "60")); + return answer; + } + + /** + * Set the configured time for input scan interval + * + * @param int value the number of minutes for input interval + */ + public static void setMinutesOfInputScanInterval(int value) { + preferences.put(INPUT_SCAN_INTERVAL_TIME, Integer.toString(value)); + } + + /** + * Copied from Autopsy UserPreferences - can be removed once everything is merged together. + * Provides ability to convert text to hex text. + */ + static final class TextConverter { + + private static final char[] TMP = "hgleri21auty84fwe".toCharArray(); //NON-NLS + private static final byte[] SALT = { + (byte) 0xde, (byte) 0x33, (byte) 0x10, (byte) 0x12, + (byte) 0xde, (byte) 0x33, (byte) 0x10, (byte) 0x12,}; + + /** + * Convert text to hex text. + * + * @param property Input text string. + * + * @return Converted hex string. + * + * @throws org.sleuthkit.autopsy.core.UserPreferencesException + */ + static String convertTextToHexText(String property) throws UserPreferencesException { + try { + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndDES"); //NON-NLS + SecretKey key = keyFactory.generateSecret(new PBEKeySpec(TMP)); + Cipher pbeCipher = Cipher.getInstance("PBEWithMD5AndDES"); //NON-NLS + pbeCipher.init(Cipher.ENCRYPT_MODE, key, new PBEParameterSpec(SALT, 20)); + return base64Encode(pbeCipher.doFinal(property.getBytes("UTF-8"))); + } catch (Exception ex) { + throw new UserPreferencesException("Error encrypting text"); + } + } + + private static String base64Encode(byte[] bytes) { + return Base64.getEncoder().encodeToString(bytes); + } + + /** + * Convert hex text back to text. + * + * @param property Input hex text string. + * + * @return Converted text string. + * + * @throws org.sleuthkit.autopsy.core.UserPreferencesException + */ + static String convertHexTextToText(String property) throws UserPreferencesException { + try { + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndDES"); //NON-NLS + SecretKey key = keyFactory.generateSecret(new PBEKeySpec(TMP)); + Cipher pbeCipher = Cipher.getInstance("PBEWithMD5AndDES"); //NON-NLS + pbeCipher.init(Cipher.DECRYPT_MODE, key, new PBEParameterSpec(SALT, 20)); + return new String(pbeCipher.doFinal(base64Decode(property)), "UTF-8"); + } catch (Exception ex) { + throw new UserPreferencesException("Error decrypting text"); + } + } + + private static byte[] base64Decode(String property) { + return Base64.getDecoder().decode(property); + } + } } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties index 46b7e0e37a..66fef4bc2d 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties @@ -95,6 +95,7 @@ AutoIngestSettingsPanel.sharedConfigCheckbox.text=Use shared configuration in fo AutoIngestSettingsPanel.jPanelSharedConfig.border.title=Shared Configuration AutoIngestSettingsPanel.cbJoinAutoIngestCluster.text=Join Automated Ingest Cluster AutoIngestSettingsPanel.tbOops.text= +AutoIngestSettingsPanel.NodeStatusLogging.text=Node Status Logging Settings AdvancedAutoIngestSettingsPanel.lbSecondsBetweenJobs.text=System synchronization wait time: AdvancedAutoIngestSettingsPanel.lbInputScanInterval.text=Interval between input scans: AdvancedAutoIngestSettingsPanel.lbRetriesAllowed.text=Maximum job retries allowed: @@ -105,3 +106,4 @@ AdvancedAutoIngestSettingsPanel.spInputScanInterval.toolTipText= AdvancedAutoIngestSettingsPanel.lbSecondsBetweenJobsSeconds.text=seconds AdvancedAutoIngestSettingsPanel.lbTimeoutHours.text=hours AdvancedAutoIngestSettingsPanel.lbInputScanIntervalMinutes.text=minutes +AutoIngestSettingsPanel.bnLogging.text=Node Status Logging diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java index a43050e2fb..46a5039ddf 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java @@ -96,6 +96,7 @@ public class SharedConfiguration { private String sharedConfigFolder; private int fileIngestThreads; private boolean sharedConfigMaster; + private boolean showToolsWarning; private boolean displayLocalTime; private boolean hideKnownFilesInDataSource; private boolean hideKnownFilesInViews; @@ -338,6 +339,8 @@ public class SharedConfiguration { private void saveNonSharedSettings() { sharedConfigMaster = AutoIngestUserPreferences.getSharedConfigMaster(); sharedConfigFolder = AutoIngestUserPreferences.getSharedConfigFolder(); + showToolsWarning = AutoIngestUserPreferences.getShowToolsWarning(); + // Get mode???? displayLocalTime = UserPreferences.displayTimesInLocalTime(); hideKnownFilesInDataSource = UserPreferences.hideKnownFilesInDataSourcesTree(); hideKnownFilesInViews = UserPreferences.hideKnownFilesInViewsTree(); @@ -351,6 +354,8 @@ public class SharedConfiguration { private void restoreNonSharedSettings() { AutoIngestUserPreferences.setSharedConfigFolder(sharedConfigFolder); AutoIngestUserPreferences.setSharedConfigMaster(sharedConfigMaster); + AutoIngestUserPreferences.setShowToolsWarning(showToolsWarning); + // Load mode??? UserPreferences.setDisplayTimesInLocalTime(displayLocalTime); UserPreferences.setHideKnownFilesInDataSourcesTree(hideKnownFilesInDataSource); UserPreferences.setHideKnownFilesInViewsTree(hideKnownFilesInViews);