19 package org.sleuthkit.autopsy.ingest;
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.Date;
24 import java.util.HashMap;
25 import java.util.List;
27 import java.util.concurrent.CopyOnWriteArrayList;
28 import java.util.concurrent.LinkedBlockingQueue;
29 import java.util.concurrent.atomic.AtomicLong;
30 import java.util.logging.Level;
31 import javax.swing.JOptionPane;
32 import org.netbeans.api.progress.ProgressHandle;
33 import org.openide.util.Cancellable;
34 import org.openide.util.NbBundle;
41 import org.
sleuthkit.datamodel.IngestJobInfo.IngestJobStatusType;
43 import org.
sleuthkit.datamodel.IngestModuleInfo.IngestModuleType;
51 final class DataSourceIngestJob {
53 private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());
59 private final IngestJob parentJob;
60 private static final AtomicLong nextJobId =
new AtomicLong(0L);
61 private final long id;
62 private final IngestJobSettings settings;
63 private final Content dataSource;
90 private final Object stageCompletionCheckLock =
new Object();
100 private final Object dataSourceIngestPipelineLock =
new Object();
101 private DataSourceIngestPipeline firstStageDataSourceIngestPipeline;
102 private DataSourceIngestPipeline secondStageDataSourceIngestPipeline;
103 private DataSourceIngestPipeline currentDataSourceIngestPipeline;
112 private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelinesQueue =
new LinkedBlockingQueue<>();
113 private final List<FileIngestPipeline> fileIngestPipelines =
new ArrayList<>();
126 private volatile boolean currentDataSourceIngestModuleCancelled;
127 private volatile boolean cancelled;
129 private final Object cancellationStateMonitor =
new Object();
130 private final List<String> cancelledDataSourceIngestModules =
new CopyOnWriteArrayList<>();
136 private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance();
142 private final boolean doUI;
148 private final Object dataSourceIngestProgressLock =
new Object();
149 private ProgressHandle dataSourceIngestProgress;
155 private final Object fileIngestProgressLock =
new Object();
156 private final List<String> filesInProgress =
new ArrayList<>();
157 private long estimatedFilesToProcess;
158 private long processedFiles;
159 private ProgressHandle fileIngestProgress;
160 private String currentFileIngestModule =
"";
161 private String currentFileIngestTask =
"";
162 private List<IngestModuleInfo> ingestModules =
new ArrayList<>();
163 private IngestJobInfo ingestJob;
168 private final long createTime;
182 this.parentJob = parentJob;
183 this.
id = DataSourceIngestJob.nextJobId.getAndIncrement();
184 this.dataSource = dataSource;
185 this.settings = settings;
186 this.doUI = runInteractively;
187 this.createTime =
new Date().getTime();
188 this.createIngestPipelines();
194 private void createIngestPipelines() {
195 List<IngestModuleTemplate> ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates();
200 Map<String, IngestModuleTemplate> dataSourceModuleTemplates =
new HashMap<>();
201 Map<String, IngestModuleTemplate> fileModuleTemplates =
new HashMap<>();
202 for (IngestModuleTemplate
template : ingestModuleTemplates) {
203 if (
template.isDataSourceIngestModuleTemplate()) {
204 dataSourceModuleTemplates.put(
template.getModuleFactory().getClass().getCanonicalName(),
template);
206 if (
template.isFileIngestModuleTemplate()) {
207 fileModuleTemplates.put(
template.getModuleFactory().getClass().getCanonicalName(),
template);
215 IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
216 List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
217 List<IngestModuleTemplate> fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
218 List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
225 for (IngestModuleTemplate
template : dataSourceModuleTemplates.values()) {
226 firstStageDataSourceModuleTemplates.add(
template);
228 for (IngestModuleTemplate
template : fileModuleTemplates.values()) {
229 fileIngestModuleTemplates.add(
template);
235 this.firstStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, firstStageDataSourceModuleTemplates);
236 this.secondStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, secondStageDataSourceModuleTemplates);
242 int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads();
243 for (
int i = 0; i < numberOfFileIngestThreads; ++i) {
244 FileIngestPipeline pipeline =
new FileIngestPipeline(
this, fileIngestModuleTemplates);
245 this.fileIngestPipelinesQueue.put(pipeline);
246 this.fileIngestPipelines.add(pipeline);
248 }
catch (InterruptedException ex) {
254 Thread.currentThread().interrupt();
256 SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase();
258 this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
259 this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase);
260 this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
261 }
catch (TskCoreException ex) {
262 logger.log(Level.SEVERE,
"Failed to add ingest modules to database.", ex);
266 private void addIngestModules(List<IngestModuleTemplate> templates, IngestModuleType type, SleuthkitCase skCase)
throws TskCoreException {
267 for (IngestModuleTemplate module : templates) {
268 ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber()));
287 private static List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(Map<String, IngestModuleTemplate> ingestModuleTemplates, List<String> pipelineConfig) {
288 List<IngestModuleTemplate> templates =
new ArrayList<>();
289 for (String moduleClassName : pipelineConfig) {
290 if (ingestModuleTemplates.containsKey(moduleClassName)) {
291 templates.add(ingestModuleTemplates.remove(moduleClassName));
311 String getExecutionContext() {
312 return this.settings.getExecutionContext();
320 Content getDataSource() {
321 return this.dataSource;
330 boolean shouldProcessUnallocatedSpace() {
331 return this.settings.getProcessUnallocatedSpace();
339 boolean hasIngestPipeline() {
340 return this.hasFirstStageDataSourceIngestPipeline()
341 || this.hasFileIngestPipeline()
342 || this.hasSecondStageDataSourceIngestPipeline();
351 private boolean hasFirstStageDataSourceIngestPipeline() {
352 return (this.firstStageDataSourceIngestPipeline.isEmpty() ==
false);
361 private boolean hasSecondStageDataSourceIngestPipeline() {
362 return (this.secondStageDataSourceIngestPipeline.isEmpty() ==
false);
370 private boolean hasFileIngestPipeline() {
371 if (!this.fileIngestPipelines.isEmpty()) {
372 return !this.fileIngestPipelines.get(0).isEmpty();
382 List<IngestModuleError> start() {
383 List<IngestModuleError> errors = startUpIngestPipelines();
384 if (errors.isEmpty()) {
385 if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) {
386 logger.log(Level.INFO,
"Starting first stage analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
387 this.startFirstStage();
388 }
else if (this.hasSecondStageDataSourceIngestPipeline()) {
389 logger.log(Level.INFO,
"Starting second stage analysis for {0} (jobId={1}), no first stage configured",
new Object[]{dataSource.getName(), this.id});
390 this.startSecondStage();
393 this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules,
new Date(this.createTime),
new Date(0), IngestJobStatusType.STARTED,
"");
394 }
catch (TskCoreException ex) {
395 logger.log(Level.SEVERE,
"Failed to add ingest job to database.", ex);
407 private List<IngestModuleError> startUpIngestPipelines() {
408 List<IngestModuleError> errors =
new ArrayList<>();
413 errors.addAll(this.firstStageDataSourceIngestPipeline.startUp());
414 errors.addAll(this.secondStageDataSourceIngestPipeline.startUp());
421 if (errors.isEmpty()) {
422 for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) {
423 errors.addAll(pipeline.startUp());
424 if (!errors.isEmpty()) {
430 while (!this.fileIngestPipelinesQueue.isEmpty()) {
431 FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll();
432 if (startedPipeline.isRunning()) {
433 List<IngestModuleError> shutDownErrors = startedPipeline.shutDown();
434 if (!shutDownErrors.isEmpty()) {
440 logIngestModuleErrors(shutDownErrors);
455 private void startFirstStage() {
456 this.stage = DataSourceIngestJob.Stages.
FIRST;
458 if (this.hasFileIngestPipeline()) {
459 synchronized (this.fileIngestProgressLock) {
460 this.estimatedFilesToProcess = this.dataSource.accept(
new GetFilesCountVisitor());
468 if (this.hasFirstStageDataSourceIngestPipeline()) {
469 this.startDataSourceIngestProgressBar();
471 if (this.hasFileIngestPipeline()) {
472 this.startFileIngestProgressBar();
480 synchronized (this.dataSourceIngestPipelineLock) {
481 this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline;
487 if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) {
488 logger.log(Level.INFO,
"Scheduling first stage data source and file level analysis tasks for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
489 DataSourceIngestJob.taskScheduler.scheduleIngestTasks(
this);
490 }
else if (this.hasFirstStageDataSourceIngestPipeline()) {
491 logger.log(Level.INFO,
"Scheduling first stage data source level analysis tasks for {0} (jobId={1}), no file level analysis configured",
new Object[]{dataSource.getName(), this.id});
492 DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(
this);
494 logger.log(Level.INFO,
"Scheduling file level analysis tasks for {0} (jobId={1}), no first stage data source level analysis configured",
new Object[]{dataSource.getName(), this.id});
495 DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(
this);
505 this.checkForStageCompleted();
512 private void startSecondStage() {
513 logger.log(Level.INFO,
"Starting second stage analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
514 this.stage = DataSourceIngestJob.Stages.
SECOND;
516 this.startDataSourceIngestProgressBar();
518 synchronized (this.dataSourceIngestPipelineLock) {
519 this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline;
521 logger.log(Level.INFO,
"Scheduling second stage data source level analysis tasks for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
522 DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(
this);
528 private void startDataSourceIngestProgressBar() {
530 synchronized (this.dataSourceIngestProgressLock) {
531 String displayName = NbBundle.getMessage(this.getClass(),
532 "IngestJob.progress.dataSourceIngest.initialDisplayName",
533 this.dataSource.getName());
534 this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
536 public boolean cancel() {
543 DataSourceIngestCancellationPanel panel =
new DataSourceIngestCancellationPanel();
544 String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(),
"IngestJob.cancellationDialog.title");
545 JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE);
546 if (panel.cancelAllDataSourceIngestModules()) {
547 DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
549 DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule();
554 this.dataSourceIngestProgress.start();
555 this.dataSourceIngestProgress.switchToIndeterminate();
563 private void startFileIngestProgressBar() {
565 synchronized (this.fileIngestProgressLock) {
566 String displayName = NbBundle.getMessage(this.getClass(),
567 "IngestJob.progress.fileIngest.displayName",
568 this.dataSource.getName());
569 this.fileIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
571 public boolean cancel() {
576 DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
580 this.fileIngestProgress.start();
581 this.fileIngestProgress.switchToDeterminate((
int) this.estimatedFilesToProcess);
590 private void checkForStageCompleted() {
591 synchronized (this.stageCompletionCheckLock) {
592 if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(
this)) {
593 switch (this.stage) {
595 this.finishFirstStage();
609 private void finishFirstStage() {
610 logger.log(Level.INFO,
"Finished first stage analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
615 List<IngestModuleError> errors =
new ArrayList<>();
616 while (!this.fileIngestPipelinesQueue.isEmpty()) {
617 FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll();
618 if (pipeline.isRunning()) {
619 errors.addAll(pipeline.shutDown());
622 if (!errors.isEmpty()) {
623 logIngestModuleErrors(errors);
629 synchronized (this.dataSourceIngestProgressLock) {
630 if (this.dataSourceIngestProgress != null) {
631 this.dataSourceIngestProgress.finish();
632 this.dataSourceIngestProgress = null;
638 synchronized (this.fileIngestProgressLock) {
639 if (this.fileIngestProgress != null) {
640 this.fileIngestProgress.finish();
641 this.fileIngestProgress = null;
649 if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) {
650 this.startSecondStage();
659 private void finish() {
660 logger.log(Level.INFO,
"Finished analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
666 synchronized (this.dataSourceIngestProgressLock) {
667 if (this.dataSourceIngestProgress != null) {
668 this.dataSourceIngestProgress.finish();
669 this.dataSourceIngestProgress = null;
673 if (this.cancelled) {
675 ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED);
676 }
catch (TskCoreException ex) {
677 logger.log(Level.SEVERE,
"Failed to set ingest status for ingest job in database.", ex);
681 ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED);
682 }
catch (TskCoreException ex) {
683 logger.log(Level.SEVERE,
"Failed to set ingest status for ingest job in database.", ex);
687 this.ingestJob.setEndDateTime(
new Date());
688 }
catch (TskCoreException ex) {
689 logger.log(Level.SEVERE,
"Failed to set end date for ingest job in database.", ex);
691 this.parentJob.dataSourceJobFinished(
this);
701 void process(DataSourceIngestTask task) {
703 synchronized (this.dataSourceIngestPipelineLock) {
704 if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
705 List<IngestModuleError> errors =
new ArrayList<>();
706 errors.addAll(this.currentDataSourceIngestPipeline.process(task));
707 if (!errors.isEmpty()) {
708 logIngestModuleErrors(errors);
718 synchronized (this.dataSourceIngestProgressLock) {
719 if (null != this.dataSourceIngestProgress) {
720 this.dataSourceIngestProgress.finish();
721 this.dataSourceIngestProgress = null;
727 DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
728 this.checkForStageCompleted();
743 void process(FileIngestTask task)
throws InterruptedException {
745 if (!this.isCancelled()) {
746 FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take();
747 if (!pipeline.isEmpty()) {
748 AbstractFile file = task.getFile();
750 synchronized (this.fileIngestProgressLock) {
751 ++this.processedFiles;
756 if (this.processedFiles <= this.estimatedFilesToProcess) {
757 this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles);
759 this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess);
761 this.filesInProgress.add(file.getName());
768 List<IngestModuleError> errors =
new ArrayList<>();
769 errors.addAll(pipeline.process(task));
770 if (!errors.isEmpty()) {
771 logIngestModuleErrors(errors);
774 if (this.doUI && !this.cancelled) {
775 synchronized (this.fileIngestProgressLock) {
780 this.filesInProgress.remove(file.getName());
781 if (this.filesInProgress.size() > 0) {
782 this.fileIngestProgress.progress(this.filesInProgress.get(0));
784 this.fileIngestProgress.progress(
"");
789 this.fileIngestPipelinesQueue.put(pipeline);
792 DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
793 this.checkForStageCompleted();
804 void addFiles(List<AbstractFile> files) {
805 if (DataSourceIngestJob.Stages.FIRST ==
this.stage) {
806 for (AbstractFile file : files) {
807 DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(
this, file);
810 DataSourceIngestJob.logger.log(Level.SEVERE,
"Adding files during second stage not supported");
819 this.checkForStageCompleted();
828 void updateDataSourceIngestProgressBarDisplayName(String displayName) {
829 if (this.doUI && !this.cancelled) {
830 synchronized (this.dataSourceIngestProgressLock) {
831 this.dataSourceIngestProgress.setDisplayName(displayName);
844 void switchDataSourceIngestProgressBarToDeterminate(
int workUnits) {
845 if (this.doUI && !this.cancelled) {
846 synchronized (this.dataSourceIngestProgressLock) {
847 if (null != this.dataSourceIngestProgress) {
848 this.dataSourceIngestProgress.switchToDeterminate(workUnits);
859 void switchDataSourceIngestProgressBarToIndeterminate() {
860 if (this.doUI && !this.cancelled) {
861 synchronized (this.dataSourceIngestProgressLock) {
862 if (null != this.dataSourceIngestProgress) {
863 this.dataSourceIngestProgress.switchToIndeterminate();
875 void advanceDataSourceIngestProgressBar(
int workUnits) {
876 if (this.doUI && !this.cancelled) {
877 synchronized (this.dataSourceIngestProgressLock) {
878 if (null != this.dataSourceIngestProgress) {
879 this.dataSourceIngestProgress.progress(
"", workUnits);
891 void advanceDataSourceIngestProgressBar(String currentTask) {
892 if (this.doUI && !this.cancelled) {
893 synchronized (this.dataSourceIngestProgressLock) {
894 if (null != this.dataSourceIngestProgress) {
895 this.dataSourceIngestProgress.progress(currentTask);
909 void advanceDataSourceIngestProgressBar(String currentTask,
int workUnits) {
910 if (this.doUI && !this.cancelled) {
911 synchronized (this.fileIngestProgressLock) {
912 this.dataSourceIngestProgress.progress(currentTask, workUnits);
924 boolean currentDataSourceIngestModuleIsCancelled() {
925 return this.currentDataSourceIngestModuleCancelled;
934 void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) {
935 this.currentDataSourceIngestModuleCancelled =
false;
936 this.cancelledDataSourceIngestModules.add(moduleDisplayName);
946 synchronized (this.dataSourceIngestProgressLock) {
947 this.dataSourceIngestProgress.finish();
948 this.dataSourceIngestProgress = null;
949 this.startDataSourceIngestProgressBar();
959 DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() {
960 if (null != this.currentDataSourceIngestPipeline) {
961 return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule();
971 void cancelCurrentDataSourceIngestModule() {
972 this.currentDataSourceIngestModuleCancelled =
true;
981 void cancel(IngestJob.CancellationReason reason) {
987 synchronized (this.dataSourceIngestProgressLock) {
988 if (dataSourceIngestProgress != null) {
989 final String displayName = NbBundle.getMessage(this.getClass(),
990 "IngestJob.progress.dataSourceIngest.initialDisplayName",
991 dataSource.getName());
992 dataSourceIngestProgress.setDisplayName(
993 NbBundle.getMessage(
this.getClass(),
994 "IngestJob.progress.cancelling",
1003 synchronized (this.fileIngestProgressLock) {
1004 if (this.fileIngestProgress != null) {
1005 final String displayName = NbBundle.getMessage(this.getClass(),
1006 "IngestJob.progress.fileIngest.displayName",
1007 this.dataSource.getName());
1008 this.fileIngestProgress.setDisplayName(
1009 NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.cancelling",
1011 if (!this.currentFileIngestModule.isEmpty() && !this.currentFileIngestTask.isEmpty()) {
1012 this.fileIngestProgress.progress(NbBundle.getMessage(
this.getClass(),
1013 "IngestJob.progress.fileIngest.cancelMessage",
1014 this.currentFileIngestModule, this.currentFileIngestTask));
1025 if (Stages.FINALIZATION != stage) {
1026 synchronized (cancellationStateMonitor) {
1031 this.cancelled =
true;
1032 this.cancellationReason = reason;
1040 DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(
this);
1041 this.checkForStageCompleted();
1051 void setCurrentFileIngestModule(String moduleName, String taskName) {
1052 this.currentFileIngestModule = moduleName;
1053 this.currentFileIngestTask = taskName;
1062 boolean isCancelled() {
1063 return this.cancelled;
1071 IngestJob.CancellationReason getCancellationReason() {
1072 return this.cancellationReason;
1080 private void logIngestModuleErrors(List<IngestModuleError> errors) {
1081 for (IngestModuleError error : errors) {
1082 DataSourceIngestJob.logger.log(Level.SEVERE, String.format(
"%s experienced an error analyzing %s (jobId=%d)", error.getModuleDisplayName(), dataSource.getName(), this.id), error.getThrowable());
1091 Snapshot getSnapshot(
boolean getIngestTasksSnapshot) {
1092 return new Snapshot(getIngestTasksSnapshot);
1098 final class Snapshot {
1100 private final String dataSource;
1101 private final long jobId;
1102 private final long jobStartTime;
1103 private final long snapShotTime;
1104 private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule;
1105 private boolean fileIngestRunning;
1106 private Date fileIngestStartTime;
1107 private final long processedFiles;
1108 private final long estimatedFilesToProcess;
1109 private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot;
1110 private final boolean jobCancelled;
1111 private final IngestJob.CancellationReason jobCancellationReason;
1112 private final List<String> cancelledDataSourceModules;
1118 Snapshot(
boolean getIngestTasksSnapshot) {
1119 this.dataSource = DataSourceIngestJob.this.dataSource.getName();
1120 this.jobId = DataSourceIngestJob.this.id;
1121 this.jobStartTime = DataSourceIngestJob.this.createTime;
1122 this.dataSourceLevelIngestModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule();
1129 for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) {
1130 if (pipeline.isRunning()) {
1131 this.fileIngestRunning =
true;
1133 Date pipelineStartTime = pipeline.getStartTime();
1134 if (null != pipelineStartTime && (null == this.fileIngestStartTime || pipelineStartTime.before(
this.fileIngestStartTime))) {
1135 this.fileIngestStartTime = pipelineStartTime;
1139 this.jobCancelled = cancelled;
1140 this.jobCancellationReason = cancellationReason;
1141 this.cancelledDataSourceModules =
new ArrayList<>(DataSourceIngestJob.this.cancelledDataSourceIngestModules);
1143 if (getIngestTasksSnapshot) {
1144 synchronized (DataSourceIngestJob.this.fileIngestProgressLock) {
1145 this.processedFiles = DataSourceIngestJob.this.processedFiles;
1146 this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess;
1147 this.snapShotTime =
new Date().getTime();
1149 this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId);
1152 this.processedFiles = 0;
1153 this.estimatedFilesToProcess = 0;
1154 this.snapShotTime =
new Date().getTime();
1155 this.tasksSnapshot = null;
1165 long getSnapshotTime() {
1166 return snapShotTime;
1175 String getDataSource() {
1195 long getJobStartTime() {
1196 return jobStartTime;
1199 DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() {
1200 return this.dataSourceLevelIngestModule;
1203 boolean fileIngestIsRunning() {
1204 return this.fileIngestRunning;
1207 Date fileIngestStartTime() {
1208 return this.fileIngestStartTime;
1218 return (
double) processedFiles / ((snapShotTime - jobStartTime) / 1000);
1226 long getFilesProcessed() {
1227 return processedFiles;
1236 long getFilesEstimated() {
1237 return estimatedFilesToProcess;
1240 long getRootQueueSize() {
1241 if (null == this.tasksSnapshot) {
1244 return this.tasksSnapshot.getRootQueueSize();
1247 long getDirQueueSize() {
1248 if (null == this.tasksSnapshot) {
1251 return this.tasksSnapshot.getDirectoryTasksQueueSize();
1254 long getFileQueueSize() {
1255 if (null == this.tasksSnapshot) {
1258 return this.tasksSnapshot.getFileQueueSize();
1261 long getDsQueueSize() {
1262 if (null == this.tasksSnapshot) {
1265 return this.tasksSnapshot.getDsQueueSize();
1268 long getRunningListSize() {
1269 if (null == this.tasksSnapshot) {
1272 return this.tasksSnapshot.getRunningListSize();
1275 boolean isCancelled() {
1276 return this.jobCancelled;
1284 IngestJob.CancellationReason getCancellationReason() {
1285 return this.jobCancellationReason;
1295 List<String> getCancelledDataSourceIngestModules() {
1296 return Collections.unmodifiableList(this.cancelledDataSourceModules);