7332 artifact pipeline work

This commit is contained in:
Richard Cordovano 2021-06-09 17:11:40 -04:00
parent dbfecb626b
commit 00852a9327
4 changed files with 42 additions and 44 deletions

View File

@ -84,13 +84,13 @@ IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.jobID=Job ID
IngestJobTableModel.colName.jobID=Job ID
IngestJobTableModel.colName.dataSource=Data Source
IngestJobTableModel.colName.start=Start
IngestJobTableModel.colName.numProcessed=Num Processed
IngestJobTableModel.colName.numProcessed=Files Processed
IngestJobTableModel.colName.filesPerSec=Files/Sec
IngestJobTableModel.colName.inProgress=In Progress
IngestJobTableModel.colName.filesQueued=Files Queued
IngestJobTableModel.colName.dirQueued=Dir Queued
IngestJobTableModel.colName.rootQueued=Root Queued
IngestJobTableModel.colName.streamingQueued=Streaming Queued
IngestJobTableModel.colName.dirQueued=Dirs Queued
IngestJobTableModel.colName.rootQueued=Roots Queued
IngestJobTableModel.colName.streamingQueued=Streamed Files Queued
IngestJobTableModel.colName.dsQueued=DS Queued
IngestJobTableModel.colName.artifactsQueued=Artifacts Queued
ModuleTableModel.colName.module=Module

View File

@ -100,13 +100,13 @@ IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.jobID=Job ID
IngestJobTableModel.colName.jobID=Job ID
IngestJobTableModel.colName.dataSource=Data Source
IngestJobTableModel.colName.start=Start
IngestJobTableModel.colName.numProcessed=Num Processed
IngestJobTableModel.colName.numProcessed=Files Processed
IngestJobTableModel.colName.filesPerSec=Files/Sec
IngestJobTableModel.colName.inProgress=In Progress
IngestJobTableModel.colName.filesQueued=Files Queued
IngestJobTableModel.colName.dirQueued=Dir Queued
IngestJobTableModel.colName.rootQueued=Root Queued
IngestJobTableModel.colName.streamingQueued=Streaming Queued
IngestJobTableModel.colName.dirQueued=Dirs Queued
IngestJobTableModel.colName.rootQueued=Roots Queued
IngestJobTableModel.colName.streamingQueued=Streamed Files Queued
IngestJobTableModel.colName.dsQueued=DS Queued
IngestJobTableModel.colName.artifactsQueued=Artifacts Queued
ModuleTableModel.colName.module=Module

View File

@ -678,7 +678,12 @@ final class IngestJobPipeline {
* GUI.
*/
if (hasFileIngestModules()) {
long filesToProcess = dataSource.accept(new GetFilesCountVisitor());;
long filesToProcess;
if (files.isEmpty()) {
filesToProcess = dataSource.accept(new GetFilesCountVisitor());
} else {
filesToProcess = files.size();
}
synchronized (fileIngestProgressLock) {
estimatedFilesToProcess = filesToProcess;
}
@ -777,20 +782,22 @@ final class IngestJobPipeline {
stage = IngestJobPipeline.Stages.FIRST_STAGE;
currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline;
/*
* Do a count of the files the data source processor has added to
* the case database. This estimate will be used for ingest progress
* snapshots and for the file ingest progress bar if running with a
* GUI.
*/
long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles;
synchronized (fileIngestProgressLock) {
if (processedFiles <= filesToProcess) {
filesToProcess -= processedFiles;
}
estimatedFilesToProcess = filesToProcess;
if (doUI && fileIngestProgressBar != null) {
fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess);
if (hasFileIngestModules()) {
/*
* Do a count of the files the data source processor has added
* to the case database. This estimate will be used for ingest
* progress snapshots and for the file ingest progress bar if
* running with a GUI.
*/
long filesToProcess = dataSource.accept(new GetFilesCountVisitor()) - processedFiles;
synchronized (fileIngestProgressLock) {
if (processedFiles <= filesToProcess) {
filesToProcess -= processedFiles;
}
estimatedFilesToProcess = filesToProcess;
if (doUI && fileIngestProgressBar != null) {
fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess);
}
}
}

View File

@ -167,28 +167,19 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel {
private static final long serialVersionUID = 1L;
private final String[] columnNames = {NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.jobID"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.dataSource"),
private final String[] columnNames = {
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.jobID"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dataSource"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.start"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.numProcessed"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.filesPerSec"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.inProgress"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.filesQueued"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.dirQueued"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.rootQueued"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.streamingQueued"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.dsQueued"),
NbBundle.getMessage(this.getClass(),
"IngestJobTableModel.colName.artifactsQueued")};
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.numProcessed"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.filesPerSec"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.inProgress"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.filesQueued"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dirQueued"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.rootQueued"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.streamingQueued"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dsQueued"),
NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.artifactsQueued")};
private List<Snapshot> jobSnapshots;