7332 artifact pipeline work

This commit is contained in:
Richard Cordovano 2021-06-09 17:11:40 -04:00
parent dbfecb626b
commit 00852a9327
4 changed files with 42 additions and 44 deletions

View File

@ -84,13 +84,13 @@ IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.jobID=Job ID
IngestJobTableModel.colName.jobID=Job ID IngestJobTableModel.colName.jobID=Job ID
IngestJobTableModel.colName.dataSource=Data Source IngestJobTableModel.colName.dataSource=Data Source
IngestJobTableModel.colName.start=Start IngestJobTableModel.colName.start=Start
IngestJobTableModel.colName.numProcessed=Num Processed IngestJobTableModel.colName.numProcessed=Files Processed
IngestJobTableModel.colName.filesPerSec=Files/Sec IngestJobTableModel.colName.filesPerSec=Files/Sec
IngestJobTableModel.colName.inProgress=In Progress IngestJobTableModel.colName.inProgress=In Progress
IngestJobTableModel.colName.filesQueued=Files Queued IngestJobTableModel.colName.filesQueued=Files Queued
IngestJobTableModel.colName.dirQueued=Dir Queued IngestJobTableModel.colName.dirQueued=Dirs Queued
IngestJobTableModel.colName.rootQueued=Root Queued IngestJobTableModel.colName.rootQueued=Roots Queued
IngestJobTableModel.colName.streamingQueued=Streaming Queued IngestJobTableModel.colName.streamingQueued=Streamed Files Queued
IngestJobTableModel.colName.dsQueued=DS Queued IngestJobTableModel.colName.dsQueued=DS Queued
IngestJobTableModel.colName.artifactsQueued=Artifacts Queued IngestJobTableModel.colName.artifactsQueued=Artifacts Queued
ModuleTableModel.colName.module=Module ModuleTableModel.colName.module=Module

View File

@ -100,13 +100,13 @@ IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.jobID=Job ID
IngestJobTableModel.colName.jobID=Job ID IngestJobTableModel.colName.jobID=Job ID
IngestJobTableModel.colName.dataSource=Data Source IngestJobTableModel.colName.dataSource=Data Source
IngestJobTableModel.colName.start=Start IngestJobTableModel.colName.start=Start
IngestJobTableModel.colName.numProcessed=Num Processed IngestJobTableModel.colName.numProcessed=Files Processed
IngestJobTableModel.colName.filesPerSec=Files/Sec IngestJobTableModel.colName.filesPerSec=Files/Sec
IngestJobTableModel.colName.inProgress=In Progress IngestJobTableModel.colName.inProgress=In Progress
IngestJobTableModel.colName.filesQueued=Files Queued IngestJobTableModel.colName.filesQueued=Files Queued
IngestJobTableModel.colName.dirQueued=Dir Queued IngestJobTableModel.colName.dirQueued=Dirs Queued
IngestJobTableModel.colName.rootQueued=Root Queued IngestJobTableModel.colName.rootQueued=Roots Queued
IngestJobTableModel.colName.streamingQueued=Streaming Queued IngestJobTableModel.colName.streamingQueued=Streamed Files Queued
IngestJobTableModel.colName.dsQueued=DS Queued IngestJobTableModel.colName.dsQueued=DS Queued
IngestJobTableModel.colName.artifactsQueued=Artifacts Queued IngestJobTableModel.colName.artifactsQueued=Artifacts Queued
ModuleTableModel.colName.module=Module ModuleTableModel.colName.module=Module

View File

@ -678,7 +678,12 @@ final class IngestJobPipeline {
* GUI. * GUI.
*/ */
if (hasFileIngestModules()) { if (hasFileIngestModules()) {
long filesToProcess = dataSource.accept(new GetFilesCountVisitor());; long filesToProcess;
if (files.isEmpty()) {
filesToProcess = dataSource.accept(new GetFilesCountVisitor());
} else {
filesToProcess = files.size();
}
synchronized (fileIngestProgressLock) { synchronized (fileIngestProgressLock) {
estimatedFilesToProcess = filesToProcess; estimatedFilesToProcess = filesToProcess;
} }
@ -777,11 +782,12 @@ final class IngestJobPipeline {
stage = IngestJobPipeline.Stages.FIRST_STAGE; stage = IngestJobPipeline.Stages.FIRST_STAGE;
currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline;
if (hasFileIngestModules()) {
/* /*
* Do a count of the files the data source processor has added to * Do a count of the files the data source processor has added
* the case database. This estimate will be used for ingest progress * to the case database. This estimate will be used for ingest
* snapshots and for the file ingest progress bar if running with a * progress snapshots and for the file ingest progress bar if
* GUI. * running with a GUI.
*/ */
long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles; long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles;
synchronized (fileIngestProgressLock) { synchronized (fileIngestProgressLock) {
@ -793,6 +799,7 @@ final class IngestJobPipeline {
fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess);
} }
} }
}
if (doUI) { if (doUI) {
if (hasFirstStageDataSourceIngestModules()) { if (hasFirstStageDataSourceIngestModules()) {

View File

@ -167,28 +167,19 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private final String[] columnNames = {NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.jobID"), private final String[] columnNames = {
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.jobID"),
"IngestJobTableModel.colName.dataSource"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dataSource"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.start"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.start"),
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.numProcessed"),
"IngestJobTableModel.colName.numProcessed"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.filesPerSec"),
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.inProgress"),
"IngestJobTableModel.colName.filesPerSec"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.filesQueued"),
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dirQueued"),
"IngestJobTableModel.colName.inProgress"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.rootQueued"),
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.streamingQueued"),
"IngestJobTableModel.colName.filesQueued"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dsQueued"),
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.artifactsQueued")};
"IngestJobTableModel.colName.dirQueued"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.rootQueued"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.streamingQueued"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.dsQueued"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.artifactsQueued")};
private List<Snapshot> jobSnapshots; private List<Snapshot> jobSnapshots;