19 package org.sleuthkit.autopsy.ingest;
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.Date;
24 import java.util.HashMap;
25 import java.util.List;
27 import java.util.concurrent.CopyOnWriteArrayList;
28 import java.util.concurrent.LinkedBlockingQueue;
29 import java.util.concurrent.atomic.AtomicLong;
30 import java.util.logging.Level;
31 import javax.swing.JOptionPane;
32 import org.netbeans.api.progress.ProgressHandle;
33 import org.openide.util.Cancellable;
34 import org.openide.util.NbBundle;
41 import org.
sleuthkit.datamodel.IngestJobInfo.IngestJobStatusType;
43 import org.
sleuthkit.datamodel.IngestModuleInfo.IngestModuleType;
52 final class DataSourceIngestJob {
54 private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());
60 private final IngestJob parentJob;
61 private static final AtomicLong nextJobId =
new AtomicLong(0L);
62 private final long id;
63 private final IngestJobSettings settings;
64 private final Content dataSource;
91 private final Object stageCompletionCheckLock =
new Object();
101 private final Object dataSourceIngestPipelineLock =
new Object();
102 private DataSourceIngestPipeline firstStageDataSourceIngestPipeline;
103 private DataSourceIngestPipeline secondStageDataSourceIngestPipeline;
104 private DataSourceIngestPipeline currentDataSourceIngestPipeline;
113 private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelinesQueue =
new LinkedBlockingQueue<>();
114 private final List<FileIngestPipeline> fileIngestPipelines =
new ArrayList<>();
127 private volatile boolean currentDataSourceIngestModuleCancelled;
128 private final List<String> cancelledDataSourceIngestModules =
new CopyOnWriteArrayList<>();
129 private volatile boolean cancelled;
136 private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance();
142 private final boolean doUI;
148 private final Object dataSourceIngestProgressLock =
new Object();
149 private ProgressHandle dataSourceIngestProgress;
155 private final Object fileIngestProgressLock =
new Object();
156 private final List<String> filesInProgress =
new ArrayList<>();
157 private long estimatedFilesToProcess;
158 private long processedFiles;
159 private ProgressHandle fileIngestProgress;
160 private String currentFileIngestModule =
"";
161 private String currentFileIngestTask =
"";
162 private final List<IngestModuleInfo> ingestModules =
new ArrayList<>();
163 private IngestJobInfo ingestJob;
168 private final long createTime;
182 this.parentJob = parentJob;
183 this.
id = DataSourceIngestJob.nextJobId.getAndIncrement();
184 this.dataSource = dataSource;
185 this.settings = settings;
186 this.doUI = runInteractively;
187 this.createTime =
new Date().getTime();
188 this.createIngestPipelines();
194 private void createIngestPipelines() {
195 List<IngestModuleTemplate> ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates();
200 Map<String, IngestModuleTemplate> dataSourceModuleTemplates =
new HashMap<>();
201 Map<String, IngestModuleTemplate> fileModuleTemplates =
new HashMap<>();
202 for (IngestModuleTemplate
template : ingestModuleTemplates) {
203 if (
template.isDataSourceIngestModuleTemplate()) {
204 dataSourceModuleTemplates.put(
template.getModuleFactory().getClass().getCanonicalName(),
template);
206 if (
template.isFileIngestModuleTemplate()) {
207 fileModuleTemplates.put(
template.getModuleFactory().getClass().getCanonicalName(),
template);
215 IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
216 List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
217 List<IngestModuleTemplate> fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
218 List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
225 for (IngestModuleTemplate
template : dataSourceModuleTemplates.values()) {
226 firstStageDataSourceModuleTemplates.add(
template);
228 for (IngestModuleTemplate
template : fileModuleTemplates.values()) {
229 fileIngestModuleTemplates.add(
template);
235 this.firstStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, firstStageDataSourceModuleTemplates);
236 this.secondStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, secondStageDataSourceModuleTemplates);
242 int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads();
243 for (
int i = 0; i < numberOfFileIngestThreads; ++i) {
244 FileIngestPipeline pipeline =
new FileIngestPipeline(
this, fileIngestModuleTemplates);
245 this.fileIngestPipelinesQueue.put(pipeline);
246 this.fileIngestPipelines.add(pipeline);
248 }
catch (InterruptedException ex) {
254 Thread.currentThread().interrupt();
256 SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase();
258 this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
259 this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase);
260 this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
261 }
catch (TskCoreException ex) {
262 logger.log(Level.SEVERE,
"Failed to add ingest modules to database.", ex);
266 private void addIngestModules(List<IngestModuleTemplate> templates, IngestModuleType type, SleuthkitCase skCase)
throws TskCoreException {
267 for (IngestModuleTemplate module : templates) {
268 ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber()));
287 private static List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(Map<String, IngestModuleTemplate> ingestModuleTemplates, List<String> pipelineConfig) {
288 List<IngestModuleTemplate> templates =
new ArrayList<>();
289 for (String moduleClassName : pipelineConfig) {
290 if (ingestModuleTemplates.containsKey(moduleClassName)) {
291 templates.add(ingestModuleTemplates.remove(moduleClassName));
311 String getExecutionContext() {
312 return this.settings.getExecutionContext();
320 Content getDataSource() {
321 return this.dataSource;
330 boolean shouldProcessUnallocatedSpace() {
331 return this.settings.getProcessUnallocatedSpace();
339 FilesSet getFileIngestFilter() {
340 return this.settings.getFileIngestFilter();
348 boolean hasIngestPipeline() {
349 return this.hasFirstStageDataSourceIngestPipeline()
350 || this.hasFileIngestPipeline()
351 || this.hasSecondStageDataSourceIngestPipeline();
360 private boolean hasFirstStageDataSourceIngestPipeline() {
361 return (this.firstStageDataSourceIngestPipeline.isEmpty() ==
false);
370 private boolean hasSecondStageDataSourceIngestPipeline() {
371 return (this.secondStageDataSourceIngestPipeline.isEmpty() ==
false);
379 private boolean hasFileIngestPipeline() {
380 if (!this.fileIngestPipelines.isEmpty()) {
381 return !this.fileIngestPipelines.get(0).isEmpty();
391 List<IngestModuleError> start() {
392 List<IngestModuleError> errors = startUpIngestPipelines();
393 if (errors.isEmpty()) {
394 if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) {
395 logger.log(Level.INFO,
"Starting first stage analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
396 this.startFirstStage();
397 }
else if (this.hasSecondStageDataSourceIngestPipeline()) {
398 logger.log(Level.INFO,
"Starting second stage analysis for {0} (jobId={1}), no first stage configured",
new Object[]{dataSource.getName(), this.id});
399 this.startSecondStage();
402 this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules,
new Date(this.createTime),
new Date(0), IngestJobStatusType.STARTED,
"");
403 }
catch (TskCoreException ex) {
404 logger.log(Level.SEVERE,
"Failed to add ingest job to database.", ex);
416 private List<IngestModuleError> startUpIngestPipelines() {
417 List<IngestModuleError> errors =
new ArrayList<>();
422 errors.addAll(this.firstStageDataSourceIngestPipeline.startUp());
423 errors.addAll(this.secondStageDataSourceIngestPipeline.startUp());
430 if (errors.isEmpty()) {
431 for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) {
432 errors.addAll(pipeline.startUp());
433 if (!errors.isEmpty()) {
439 while (!this.fileIngestPipelinesQueue.isEmpty()) {
440 FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll();
441 if (startedPipeline.isRunning()) {
442 List<IngestModuleError> shutDownErrors = startedPipeline.shutDown();
443 if (!shutDownErrors.isEmpty()) {
449 logIngestModuleErrors(shutDownErrors);
464 private void startFirstStage() {
465 this.stage = DataSourceIngestJob.Stages.
FIRST;
467 if (this.hasFileIngestPipeline()) {
468 synchronized (this.fileIngestProgressLock) {
469 this.estimatedFilesToProcess = this.dataSource.accept(
new GetFilesCountVisitor());
477 if (this.hasFirstStageDataSourceIngestPipeline()) {
478 this.startDataSourceIngestProgressBar();
480 if (this.hasFileIngestPipeline()) {
481 this.startFileIngestProgressBar();
489 synchronized (this.dataSourceIngestPipelineLock) {
490 this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline;
496 if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) {
497 logger.log(Level.INFO,
"Scheduling first stage data source and file level analysis tasks for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
498 DataSourceIngestJob.taskScheduler.scheduleIngestTasks(
this);
499 }
else if (this.hasFirstStageDataSourceIngestPipeline()) {
500 logger.log(Level.INFO,
"Scheduling first stage data source level analysis tasks for {0} (jobId={1}), no file level analysis configured",
new Object[]{dataSource.getName(), this.id});
501 DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(
this);
503 logger.log(Level.INFO,
"Scheduling file level analysis tasks for {0} (jobId={1}), no first stage data source level analysis configured",
new Object[]{dataSource.getName(), this.id});
504 DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(
this);
514 this.checkForStageCompleted();
521 private void startSecondStage() {
522 logger.log(Level.INFO,
"Starting second stage analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
523 this.stage = DataSourceIngestJob.Stages.
SECOND;
525 this.startDataSourceIngestProgressBar();
527 synchronized (this.dataSourceIngestPipelineLock) {
528 this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline;
530 logger.log(Level.INFO,
"Scheduling second stage data source level analysis tasks for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
531 DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(
this);
537 private void startDataSourceIngestProgressBar() {
539 synchronized (this.dataSourceIngestProgressLock) {
540 String displayName = NbBundle.getMessage(this.getClass(),
541 "IngestJob.progress.dataSourceIngest.initialDisplayName",
542 this.dataSource.getName());
543 this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
545 public boolean cancel() {
552 DataSourceIngestCancellationPanel panel =
new DataSourceIngestCancellationPanel();
553 String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(),
"IngestJob.cancellationDialog.title");
554 JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE);
555 if (panel.cancelAllDataSourceIngestModules()) {
556 DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
558 DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule();
563 this.dataSourceIngestProgress.start();
564 this.dataSourceIngestProgress.switchToIndeterminate();
572 private void startFileIngestProgressBar() {
574 synchronized (this.fileIngestProgressLock) {
575 String displayName = NbBundle.getMessage(this.getClass(),
576 "IngestJob.progress.fileIngest.displayName",
577 this.dataSource.getName());
578 this.fileIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
580 public boolean cancel() {
585 DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
589 this.fileIngestProgress.start();
590 this.fileIngestProgress.switchToDeterminate((
int) this.estimatedFilesToProcess);
599 private void checkForStageCompleted() {
600 synchronized (this.stageCompletionCheckLock) {
601 if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(
this)) {
602 switch (this.stage) {
604 this.finishFirstStage();
618 private void finishFirstStage() {
619 logger.log(Level.INFO,
"Finished first stage analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
624 List<IngestModuleError> errors =
new ArrayList<>();
625 while (!this.fileIngestPipelinesQueue.isEmpty()) {
626 FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll();
627 if (pipeline.isRunning()) {
628 errors.addAll(pipeline.shutDown());
631 if (!errors.isEmpty()) {
632 logIngestModuleErrors(errors);
638 synchronized (this.dataSourceIngestProgressLock) {
639 if (this.dataSourceIngestProgress != null) {
640 this.dataSourceIngestProgress.finish();
641 this.dataSourceIngestProgress = null;
647 synchronized (this.fileIngestProgressLock) {
648 if (this.fileIngestProgress != null) {
649 this.fileIngestProgress.finish();
650 this.fileIngestProgress = null;
658 if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) {
659 this.startSecondStage();
668 private void finish() {
669 logger.log(Level.INFO,
"Finished analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
675 synchronized (this.dataSourceIngestProgressLock) {
676 if (this.dataSourceIngestProgress != null) {
677 this.dataSourceIngestProgress.finish();
678 this.dataSourceIngestProgress = null;
682 if (this.cancelled) {
684 ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED);
685 }
catch (TskCoreException ex) {
686 logger.log(Level.SEVERE,
"Failed to set ingest status for ingest job in database.", ex);
690 ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED);
691 }
catch (TskCoreException ex) {
692 logger.log(Level.SEVERE,
"Failed to set ingest status for ingest job in database.", ex);
696 this.ingestJob.setEndDateTime(
new Date());
697 }
catch (TskCoreException ex) {
698 logger.log(Level.SEVERE,
"Failed to set end date for ingest job in database.", ex);
700 this.parentJob.dataSourceJobFinished(
this);
710 void process(DataSourceIngestTask task) {
712 synchronized (this.dataSourceIngestPipelineLock) {
713 if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
714 List<IngestModuleError> errors =
new ArrayList<>();
715 errors.addAll(this.currentDataSourceIngestPipeline.process(task));
716 if (!errors.isEmpty()) {
717 logIngestModuleErrors(errors);
727 synchronized (this.dataSourceIngestProgressLock) {
728 if (null != this.dataSourceIngestProgress) {
729 this.dataSourceIngestProgress.finish();
730 this.dataSourceIngestProgress = null;
736 DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
737 this.checkForStageCompleted();
752 void process(FileIngestTask task)
throws InterruptedException {
754 if (!this.isCancelled()) {
755 FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take();
756 if (!pipeline.isEmpty()) {
757 AbstractFile file = task.getFile();
759 synchronized (this.fileIngestProgressLock) {
760 ++this.processedFiles;
765 if (this.processedFiles <= this.estimatedFilesToProcess) {
766 this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles);
768 this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess);
770 this.filesInProgress.add(file.getName());
777 List<IngestModuleError> errors =
new ArrayList<>();
778 errors.addAll(pipeline.process(task));
779 if (!errors.isEmpty()) {
780 logIngestModuleErrors(errors);
783 if (this.doUI && !this.cancelled) {
784 synchronized (this.fileIngestProgressLock) {
789 this.filesInProgress.remove(file.getName());
790 if (this.filesInProgress.size() > 0) {
791 this.fileIngestProgress.progress(this.filesInProgress.get(0));
793 this.fileIngestProgress.progress(
"");
798 this.fileIngestPipelinesQueue.put(pipeline);
801 DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
802 this.checkForStageCompleted();
813 void addFiles(List<AbstractFile> files) {
814 if (DataSourceIngestJob.Stages.FIRST ==
this.stage) {
815 for (AbstractFile file : files) {
816 DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(
this, file);
819 DataSourceIngestJob.logger.log(Level.SEVERE,
"Adding files during second stage not supported");
828 this.checkForStageCompleted();
837 void updateDataSourceIngestProgressBarDisplayName(String displayName) {
838 if (this.doUI && !this.cancelled) {
839 synchronized (this.dataSourceIngestProgressLock) {
840 this.dataSourceIngestProgress.setDisplayName(displayName);
853 void switchDataSourceIngestProgressBarToDeterminate(
int workUnits) {
854 if (this.doUI && !this.cancelled) {
855 synchronized (this.dataSourceIngestProgressLock) {
856 if (null != this.dataSourceIngestProgress) {
857 this.dataSourceIngestProgress.switchToDeterminate(workUnits);
868 void switchDataSourceIngestProgressBarToIndeterminate() {
869 if (this.doUI && !this.cancelled) {
870 synchronized (this.dataSourceIngestProgressLock) {
871 if (null != this.dataSourceIngestProgress) {
872 this.dataSourceIngestProgress.switchToIndeterminate();
884 void advanceDataSourceIngestProgressBar(
int workUnits) {
885 if (this.doUI && !this.cancelled) {
886 synchronized (this.dataSourceIngestProgressLock) {
887 if (null != this.dataSourceIngestProgress) {
888 this.dataSourceIngestProgress.progress(
"", workUnits);
900 void advanceDataSourceIngestProgressBar(String currentTask) {
901 if (this.doUI && !this.cancelled) {
902 synchronized (this.dataSourceIngestProgressLock) {
903 if (null != this.dataSourceIngestProgress) {
904 this.dataSourceIngestProgress.progress(currentTask);
918 void advanceDataSourceIngestProgressBar(String currentTask,
int workUnits) {
919 if (this.doUI && !this.cancelled) {
920 synchronized (this.fileIngestProgressLock) {
921 this.dataSourceIngestProgress.progress(currentTask, workUnits);
933 boolean currentDataSourceIngestModuleIsCancelled() {
934 return this.currentDataSourceIngestModuleCancelled;
943 void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) {
944 this.currentDataSourceIngestModuleCancelled =
false;
945 this.cancelledDataSourceIngestModules.add(moduleDisplayName);
955 synchronized (this.dataSourceIngestProgressLock) {
956 this.dataSourceIngestProgress.finish();
957 this.dataSourceIngestProgress = null;
958 this.startDataSourceIngestProgressBar();
968 DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() {
969 if (null != this.currentDataSourceIngestPipeline) {
970 return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule();
980 void cancelCurrentDataSourceIngestModule() {
981 this.currentDataSourceIngestModuleCancelled =
true;
990 void cancel(IngestJob.CancellationReason reason) {
991 this.cancelled =
true;
992 this.cancellationReason = reason;
993 DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(
this);
996 synchronized (this.dataSourceIngestProgressLock) {
997 if (null != dataSourceIngestProgress) {
998 dataSourceIngestProgress.setDisplayName(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName()));
999 dataSourceIngestProgress.progress(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.cancelling"));
1003 synchronized (this.fileIngestProgressLock) {
1004 if (null != this.fileIngestProgress) {
1005 this.fileIngestProgress.setDisplayName(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.fileIngest.displayName", this.dataSource.getName()));
1006 this.fileIngestProgress.progress(NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.cancelling"));
1019 void setCurrentFileIngestModule(String moduleName, String taskName) {
1020 this.currentFileIngestModule = moduleName;
1021 this.currentFileIngestTask = taskName;
1030 boolean isCancelled() {
1031 return this.cancelled;
1039 IngestJob.CancellationReason getCancellationReason() {
1040 return this.cancellationReason;
1048 private void logIngestModuleErrors(List<IngestModuleError> errors) {
1049 for (IngestModuleError error : errors) {
1050 DataSourceIngestJob.logger.log(Level.SEVERE, String.format(
"%s experienced an error analyzing %s (jobId=%d)", error.getModuleDisplayName(), dataSource.getName(), this.id), error.getThrowable());
1059 Snapshot getSnapshot(
boolean getIngestTasksSnapshot) {
1060 return new Snapshot(getIngestTasksSnapshot);
1066 final class Snapshot {
1068 private final String dataSource;
1069 private final long jobId;
1070 private final long jobStartTime;
1071 private final long snapShotTime;
1072 private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule;
1073 private boolean fileIngestRunning;
1074 private Date fileIngestStartTime;
1075 private final long processedFiles;
1076 private final long estimatedFilesToProcess;
1077 private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot;
1078 private final boolean jobCancelled;
1079 private final IngestJob.CancellationReason jobCancellationReason;
1080 private final List<String> cancelledDataSourceModules;
1086 Snapshot(
boolean getIngestTasksSnapshot) {
1087 this.dataSource = DataSourceIngestJob.this.dataSource.getName();
1088 this.jobId = DataSourceIngestJob.this.id;
1089 this.jobStartTime = DataSourceIngestJob.this.createTime;
1090 this.dataSourceLevelIngestModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule();
1097 for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) {
1098 if (pipeline.isRunning()) {
1099 this.fileIngestRunning =
true;
1101 Date pipelineStartTime = pipeline.getStartTime();
1102 if (null != pipelineStartTime && (null == this.fileIngestStartTime || pipelineStartTime.before(
this.fileIngestStartTime))) {
1103 this.fileIngestStartTime = pipelineStartTime;
1107 this.jobCancelled = cancelled;
1108 this.jobCancellationReason = cancellationReason;
1109 this.cancelledDataSourceModules =
new ArrayList<>(DataSourceIngestJob.this.cancelledDataSourceIngestModules);
1111 if (getIngestTasksSnapshot) {
1112 synchronized (DataSourceIngestJob.this.fileIngestProgressLock) {
1113 this.processedFiles = DataSourceIngestJob.this.processedFiles;
1114 this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess;
1115 this.snapShotTime =
new Date().getTime();
1117 this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId);
1120 this.processedFiles = 0;
1121 this.estimatedFilesToProcess = 0;
1122 this.snapShotTime =
new Date().getTime();
1123 this.tasksSnapshot = null;
1133 long getSnapshotTime() {
1134 return snapShotTime;
1143 String getDataSource() {
1163 long getJobStartTime() {
1164 return jobStartTime;
1167 DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() {
1168 return this.dataSourceLevelIngestModule;
1171 boolean fileIngestIsRunning() {
1172 return this.fileIngestRunning;
1175 Date fileIngestStartTime() {
1176 return this.fileIngestStartTime;
1186 return (
double) processedFiles / ((snapShotTime - jobStartTime) / 1000);
1194 long getFilesProcessed() {
1195 return processedFiles;
1204 long getFilesEstimated() {
1205 return estimatedFilesToProcess;
1208 long getRootQueueSize() {
1209 if (null == this.tasksSnapshot) {
1212 return this.tasksSnapshot.getRootQueueSize();
1215 long getDirQueueSize() {
1216 if (null == this.tasksSnapshot) {
1219 return this.tasksSnapshot.getDirectoryTasksQueueSize();
1222 long getFileQueueSize() {
1223 if (null == this.tasksSnapshot) {
1226 return this.tasksSnapshot.getFileQueueSize();
1229 long getDsQueueSize() {
1230 if (null == this.tasksSnapshot) {
1233 return this.tasksSnapshot.getDsQueueSize();
1236 long getRunningListSize() {
1237 if (null == this.tasksSnapshot) {
1240 return this.tasksSnapshot.getRunningListSize();
1243 boolean isCancelled() {
1244 return this.jobCancelled;
1252 IngestJob.CancellationReason getCancellationReason() {
1253 return this.jobCancellationReason;
1263 List<String> getCancelledDataSourceIngestModules() {
1264 return Collections.unmodifiableList(this.cancelledDataSourceModules);