19 package org.sleuthkit.autopsy.ingest;
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.Date;
24 import java.util.HashMap;
25 import java.util.List;
27 import java.util.concurrent.CopyOnWriteArrayList;
28 import java.util.concurrent.LinkedBlockingQueue;
29 import java.util.concurrent.atomic.AtomicLong;
30 import java.util.logging.Level;
31 import javax.swing.JOptionPane;
32 import org.netbeans.api.progress.ProgressHandle;
33 import org.openide.util.Cancellable;
34 import org.openide.util.NbBundle;
41 import org.
sleuthkit.datamodel.IngestJobInfo.IngestJobStatusType;
43 import org.
sleuthkit.datamodel.IngestModuleInfo.IngestModuleType;
52 final class DataSourceIngestJob {
54 private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());
60 private final IngestJob parentJob;
61 private static final AtomicLong nextJobId =
new AtomicLong(0L);
62 private final long id;
63 private final IngestJobSettings settings;
64 private final Content dataSource;
91 private final Object stageCompletionCheckLock =
new Object();
101 private final Object dataSourceIngestPipelineLock =
new Object();
102 private DataSourceIngestPipeline firstStageDataSourceIngestPipeline;
103 private DataSourceIngestPipeline secondStageDataSourceIngestPipeline;
104 private DataSourceIngestPipeline currentDataSourceIngestPipeline;
113 private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelinesQueue =
new LinkedBlockingQueue<>();
114 private final List<FileIngestPipeline> fileIngestPipelines =
new ArrayList<>();
127 private volatile boolean currentDataSourceIngestModuleCancelled;
128 private volatile boolean cancelled;
130 private final Object cancellationStateMonitor =
new Object();
131 private final List<String> cancelledDataSourceIngestModules =
new CopyOnWriteArrayList<>();
137 private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance();
143 private final boolean doUI;
149 private final Object dataSourceIngestProgressLock =
new Object();
150 private ProgressHandle dataSourceIngestProgress;
156 private final Object fileIngestProgressLock =
new Object();
157 private final List<String> filesInProgress =
new ArrayList<>();
158 private long estimatedFilesToProcess;
159 private long processedFiles;
160 private ProgressHandle fileIngestProgress;
161 private String currentFileIngestModule =
"";
162 private String currentFileIngestTask =
"";
163 private final List<IngestModuleInfo> ingestModules =
new ArrayList<>();
164 private IngestJobInfo ingestJob;
169 private final long createTime;
183 this.parentJob = parentJob;
184 this.
id = DataSourceIngestJob.nextJobId.getAndIncrement();
185 this.dataSource = dataSource;
186 this.settings = settings;
187 this.doUI = runInteractively;
188 this.createTime =
new Date().getTime();
189 this.createIngestPipelines();
195 private void createIngestPipelines() {
196 List<IngestModuleTemplate> ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates();
201 Map<String, IngestModuleTemplate> dataSourceModuleTemplates =
new HashMap<>();
202 Map<String, IngestModuleTemplate> fileModuleTemplates =
new HashMap<>();
203 for (IngestModuleTemplate
template : ingestModuleTemplates) {
204 if (
template.isDataSourceIngestModuleTemplate()) {
205 dataSourceModuleTemplates.put(
template.getModuleFactory().getClass().getCanonicalName(),
template);
207 if (
template.isFileIngestModuleTemplate()) {
208 fileModuleTemplates.put(
template.getModuleFactory().getClass().getCanonicalName(),
template);
216 IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
217 List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
218 List<IngestModuleTemplate> fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
219 List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
226 for (IngestModuleTemplate
template : dataSourceModuleTemplates.values()) {
227 firstStageDataSourceModuleTemplates.add(
template);
229 for (IngestModuleTemplate
template : fileModuleTemplates.values()) {
230 fileIngestModuleTemplates.add(
template);
236 this.firstStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, firstStageDataSourceModuleTemplates);
237 this.secondStageDataSourceIngestPipeline =
new DataSourceIngestPipeline(
this, secondStageDataSourceModuleTemplates);
243 int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads();
244 for (
int i = 0; i < numberOfFileIngestThreads; ++i) {
245 FileIngestPipeline pipeline =
new FileIngestPipeline(
this, fileIngestModuleTemplates);
246 this.fileIngestPipelinesQueue.put(pipeline);
247 this.fileIngestPipelines.add(pipeline);
249 }
catch (InterruptedException ex) {
255 Thread.currentThread().interrupt();
257 SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase();
259 this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
260 this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase);
261 this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
262 }
catch (TskCoreException ex) {
263 logger.log(Level.SEVERE,
"Failed to add ingest modules to database.", ex);
267 private void addIngestModules(List<IngestModuleTemplate> templates, IngestModuleType type, SleuthkitCase skCase)
throws TskCoreException {
268 for (IngestModuleTemplate module : templates) {
269 ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber()));
288 private static List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(Map<String, IngestModuleTemplate> ingestModuleTemplates, List<String> pipelineConfig) {
289 List<IngestModuleTemplate> templates =
new ArrayList<>();
290 for (String moduleClassName : pipelineConfig) {
291 if (ingestModuleTemplates.containsKey(moduleClassName)) {
292 templates.add(ingestModuleTemplates.remove(moduleClassName));
312 String getExecutionContext() {
313 return this.settings.getExecutionContext();
321 Content getDataSource() {
322 return this.dataSource;
331 boolean shouldProcessUnallocatedSpace() {
332 return this.settings.getProcessUnallocatedSpace();
340 FilesSet getFileIngestFilter() {
341 return this.settings.getFileIngestFilter();
349 boolean hasIngestPipeline() {
350 return this.hasFirstStageDataSourceIngestPipeline()
351 || this.hasFileIngestPipeline()
352 || this.hasSecondStageDataSourceIngestPipeline();
361 private boolean hasFirstStageDataSourceIngestPipeline() {
362 return (this.firstStageDataSourceIngestPipeline.isEmpty() ==
false);
371 private boolean hasSecondStageDataSourceIngestPipeline() {
372 return (this.secondStageDataSourceIngestPipeline.isEmpty() ==
false);
380 private boolean hasFileIngestPipeline() {
381 if (!this.fileIngestPipelines.isEmpty()) {
382 return !this.fileIngestPipelines.get(0).isEmpty();
392 List<IngestModuleError> start() {
393 List<IngestModuleError> errors = startUpIngestPipelines();
394 if (errors.isEmpty()) {
395 if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) {
396 logger.log(Level.INFO,
"Starting first stage analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
397 this.startFirstStage();
398 }
else if (this.hasSecondStageDataSourceIngestPipeline()) {
399 logger.log(Level.INFO,
"Starting second stage analysis for {0} (jobId={1}), no first stage configured",
new Object[]{dataSource.getName(), this.id});
400 this.startSecondStage();
403 this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules,
new Date(this.createTime),
new Date(0), IngestJobStatusType.STARTED,
"");
404 }
catch (TskCoreException ex) {
405 logger.log(Level.SEVERE,
"Failed to add ingest job to database.", ex);
417 private List<IngestModuleError> startUpIngestPipelines() {
418 List<IngestModuleError> errors =
new ArrayList<>();
423 errors.addAll(this.firstStageDataSourceIngestPipeline.startUp());
424 errors.addAll(this.secondStageDataSourceIngestPipeline.startUp());
431 if (errors.isEmpty()) {
432 for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) {
433 errors.addAll(pipeline.startUp());
434 if (!errors.isEmpty()) {
440 while (!this.fileIngestPipelinesQueue.isEmpty()) {
441 FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll();
442 if (startedPipeline.isRunning()) {
443 List<IngestModuleError> shutDownErrors = startedPipeline.shutDown();
444 if (!shutDownErrors.isEmpty()) {
450 logIngestModuleErrors(shutDownErrors);
465 private void startFirstStage() {
466 this.stage = DataSourceIngestJob.Stages.
FIRST;
468 if (this.hasFileIngestPipeline()) {
469 synchronized (this.fileIngestProgressLock) {
470 this.estimatedFilesToProcess = this.dataSource.accept(
new GetFilesCountVisitor());
478 if (this.hasFirstStageDataSourceIngestPipeline()) {
479 this.startDataSourceIngestProgressBar();
481 if (this.hasFileIngestPipeline()) {
482 this.startFileIngestProgressBar();
490 synchronized (this.dataSourceIngestPipelineLock) {
491 this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline;
497 if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) {
498 logger.log(Level.INFO,
"Scheduling first stage data source and file level analysis tasks for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
499 DataSourceIngestJob.taskScheduler.scheduleIngestTasks(
this);
500 }
else if (this.hasFirstStageDataSourceIngestPipeline()) {
501 logger.log(Level.INFO,
"Scheduling first stage data source level analysis tasks for {0} (jobId={1}), no file level analysis configured",
new Object[]{dataSource.getName(), this.id});
502 DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(
this);
504 logger.log(Level.INFO,
"Scheduling file level analysis tasks for {0} (jobId={1}), no first stage data source level analysis configured",
new Object[]{dataSource.getName(), this.id});
505 DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(
this);
515 this.checkForStageCompleted();
522 private void startSecondStage() {
523 logger.log(Level.INFO,
"Starting second stage analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
524 this.stage = DataSourceIngestJob.Stages.
SECOND;
526 this.startDataSourceIngestProgressBar();
528 synchronized (this.dataSourceIngestPipelineLock) {
529 this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline;
531 logger.log(Level.INFO,
"Scheduling second stage data source level analysis tasks for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
532 DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(
this);
538 private void startDataSourceIngestProgressBar() {
540 synchronized (this.dataSourceIngestProgressLock) {
541 String displayName = NbBundle.getMessage(this.getClass(),
542 "IngestJob.progress.dataSourceIngest.initialDisplayName",
543 this.dataSource.getName());
544 this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
546 public boolean cancel() {
553 DataSourceIngestCancellationPanel panel =
new DataSourceIngestCancellationPanel();
554 String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(),
"IngestJob.cancellationDialog.title");
555 JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE);
556 if (panel.cancelAllDataSourceIngestModules()) {
557 DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
559 DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule();
564 this.dataSourceIngestProgress.start();
565 this.dataSourceIngestProgress.switchToIndeterminate();
573 private void startFileIngestProgressBar() {
575 synchronized (this.fileIngestProgressLock) {
576 String displayName = NbBundle.getMessage(this.getClass(),
577 "IngestJob.progress.fileIngest.displayName",
578 this.dataSource.getName());
579 this.fileIngestProgress = ProgressHandle.createHandle(displayName,
new Cancellable() {
581 public boolean cancel() {
586 DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
590 this.fileIngestProgress.start();
591 this.fileIngestProgress.switchToDeterminate((
int) this.estimatedFilesToProcess);
600 private void checkForStageCompleted() {
601 synchronized (this.stageCompletionCheckLock) {
602 if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(
this)) {
603 switch (this.stage) {
605 this.finishFirstStage();
619 private void finishFirstStage() {
620 logger.log(Level.INFO,
"Finished first stage analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
625 List<IngestModuleError> errors =
new ArrayList<>();
626 while (!this.fileIngestPipelinesQueue.isEmpty()) {
627 FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll();
628 if (pipeline.isRunning()) {
629 errors.addAll(pipeline.shutDown());
632 if (!errors.isEmpty()) {
633 logIngestModuleErrors(errors);
639 synchronized (this.dataSourceIngestProgressLock) {
640 if (this.dataSourceIngestProgress != null) {
641 this.dataSourceIngestProgress.finish();
642 this.dataSourceIngestProgress = null;
648 synchronized (this.fileIngestProgressLock) {
649 if (this.fileIngestProgress != null) {
650 this.fileIngestProgress.finish();
651 this.fileIngestProgress = null;
659 if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) {
660 this.startSecondStage();
669 private void finish() {
670 logger.log(Level.INFO,
"Finished analysis for {0} (jobId={1})",
new Object[]{dataSource.getName(), this.id});
676 synchronized (this.dataSourceIngestProgressLock) {
677 if (this.dataSourceIngestProgress != null) {
678 this.dataSourceIngestProgress.finish();
679 this.dataSourceIngestProgress = null;
683 if (this.cancelled) {
685 ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED);
686 }
catch (TskCoreException ex) {
687 logger.log(Level.SEVERE,
"Failed to set ingest status for ingest job in database.", ex);
691 ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED);
692 }
catch (TskCoreException ex) {
693 logger.log(Level.SEVERE,
"Failed to set ingest status for ingest job in database.", ex);
697 this.ingestJob.setEndDateTime(
new Date());
698 }
catch (TskCoreException ex) {
699 logger.log(Level.SEVERE,
"Failed to set end date for ingest job in database.", ex);
701 this.parentJob.dataSourceJobFinished(
this);
711 void process(DataSourceIngestTask task) {
713 synchronized (this.dataSourceIngestPipelineLock) {
714 if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
715 List<IngestModuleError> errors =
new ArrayList<>();
716 errors.addAll(this.currentDataSourceIngestPipeline.process(task));
717 if (!errors.isEmpty()) {
718 logIngestModuleErrors(errors);
728 synchronized (this.dataSourceIngestProgressLock) {
729 if (null != this.dataSourceIngestProgress) {
730 this.dataSourceIngestProgress.finish();
731 this.dataSourceIngestProgress = null;
737 DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
738 this.checkForStageCompleted();
753 void process(FileIngestTask task)
throws InterruptedException {
755 if (!this.isCancelled()) {
756 FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take();
757 if (!pipeline.isEmpty()) {
758 AbstractFile file = task.getFile();
760 synchronized (this.fileIngestProgressLock) {
761 ++this.processedFiles;
766 if (this.processedFiles <= this.estimatedFilesToProcess) {
767 this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles);
769 this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess);
771 this.filesInProgress.add(file.getName());
778 List<IngestModuleError> errors =
new ArrayList<>();
779 errors.addAll(pipeline.process(task));
780 if (!errors.isEmpty()) {
781 logIngestModuleErrors(errors);
784 if (this.doUI && !this.cancelled) {
785 synchronized (this.fileIngestProgressLock) {
790 this.filesInProgress.remove(file.getName());
791 if (this.filesInProgress.size() > 0) {
792 this.fileIngestProgress.progress(this.filesInProgress.get(0));
794 this.fileIngestProgress.progress(
"");
799 this.fileIngestPipelinesQueue.put(pipeline);
802 DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
803 this.checkForStageCompleted();
814 void addFiles(List<AbstractFile> files) {
815 if (DataSourceIngestJob.Stages.FIRST ==
this.stage) {
816 for (AbstractFile file : files) {
817 DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(
this, file);
820 DataSourceIngestJob.logger.log(Level.SEVERE,
"Adding files during second stage not supported");
829 this.checkForStageCompleted();
838 void updateDataSourceIngestProgressBarDisplayName(String displayName) {
839 if (this.doUI && !this.cancelled) {
840 synchronized (this.dataSourceIngestProgressLock) {
841 this.dataSourceIngestProgress.setDisplayName(displayName);
854 void switchDataSourceIngestProgressBarToDeterminate(
int workUnits) {
855 if (this.doUI && !this.cancelled) {
856 synchronized (this.dataSourceIngestProgressLock) {
857 if (null != this.dataSourceIngestProgress) {
858 this.dataSourceIngestProgress.switchToDeterminate(workUnits);
869 void switchDataSourceIngestProgressBarToIndeterminate() {
870 if (this.doUI && !this.cancelled) {
871 synchronized (this.dataSourceIngestProgressLock) {
872 if (null != this.dataSourceIngestProgress) {
873 this.dataSourceIngestProgress.switchToIndeterminate();
885 void advanceDataSourceIngestProgressBar(
int workUnits) {
886 if (this.doUI && !this.cancelled) {
887 synchronized (this.dataSourceIngestProgressLock) {
888 if (null != this.dataSourceIngestProgress) {
889 this.dataSourceIngestProgress.progress(
"", workUnits);
901 void advanceDataSourceIngestProgressBar(String currentTask) {
902 if (this.doUI && !this.cancelled) {
903 synchronized (this.dataSourceIngestProgressLock) {
904 if (null != this.dataSourceIngestProgress) {
905 this.dataSourceIngestProgress.progress(currentTask);
919 void advanceDataSourceIngestProgressBar(String currentTask,
int workUnits) {
920 if (this.doUI && !this.cancelled) {
921 synchronized (this.fileIngestProgressLock) {
922 this.dataSourceIngestProgress.progress(currentTask, workUnits);
934 boolean currentDataSourceIngestModuleIsCancelled() {
935 return this.currentDataSourceIngestModuleCancelled;
944 void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) {
945 this.currentDataSourceIngestModuleCancelled =
false;
946 this.cancelledDataSourceIngestModules.add(moduleDisplayName);
956 synchronized (this.dataSourceIngestProgressLock) {
957 this.dataSourceIngestProgress.finish();
958 this.dataSourceIngestProgress = null;
959 this.startDataSourceIngestProgressBar();
969 DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() {
970 if (null != this.currentDataSourceIngestPipeline) {
971 return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule();
981 void cancelCurrentDataSourceIngestModule() {
982 this.currentDataSourceIngestModuleCancelled =
true;
991 void cancel(IngestJob.CancellationReason reason) {
997 synchronized (this.dataSourceIngestProgressLock) {
998 if (dataSourceIngestProgress != null) {
999 final String displayName = NbBundle.getMessage(this.getClass(),
1000 "IngestJob.progress.dataSourceIngest.initialDisplayName",
1001 dataSource.getName());
1002 dataSourceIngestProgress.setDisplayName(
1003 NbBundle.getMessage(
this.getClass(),
1004 "IngestJob.progress.cancelling",
1013 synchronized (this.fileIngestProgressLock) {
1014 if (this.fileIngestProgress != null) {
1015 final String displayName = NbBundle.getMessage(this.getClass(),
1016 "IngestJob.progress.fileIngest.displayName",
1017 this.dataSource.getName());
1018 this.fileIngestProgress.setDisplayName(
1019 NbBundle.getMessage(
this.getClass(),
"IngestJob.progress.cancelling",
1021 if (!this.currentFileIngestModule.isEmpty() && !this.currentFileIngestTask.isEmpty()) {
1022 this.fileIngestProgress.progress(NbBundle.getMessage(
this.getClass(),
1023 "IngestJob.progress.fileIngest.cancelMessage",
1024 this.currentFileIngestModule, this.currentFileIngestTask));
1035 if (Stages.FINALIZATION != stage) {
1036 synchronized (cancellationStateMonitor) {
1041 this.cancelled =
true;
1042 this.cancellationReason = reason;
1050 DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(
this);
1051 this.checkForStageCompleted();
1061 void setCurrentFileIngestModule(String moduleName, String taskName) {
1062 this.currentFileIngestModule = moduleName;
1063 this.currentFileIngestTask = taskName;
1072 boolean isCancelled() {
1073 return this.cancelled;
1081 IngestJob.CancellationReason getCancellationReason() {
1082 return this.cancellationReason;
1090 private void logIngestModuleErrors(List<IngestModuleError> errors) {
1091 for (IngestModuleError error : errors) {
1092 DataSourceIngestJob.logger.log(Level.SEVERE, String.format(
"%s experienced an error analyzing %s (jobId=%d)", error.getModuleDisplayName(), dataSource.getName(), this.id), error.getThrowable());
1101 Snapshot getSnapshot(
boolean getIngestTasksSnapshot) {
1102 return new Snapshot(getIngestTasksSnapshot);
1108 final class Snapshot {
1110 private final String dataSource;
1111 private final long jobId;
1112 private final long jobStartTime;
1113 private final long snapShotTime;
1114 private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule;
1115 private boolean fileIngestRunning;
1116 private Date fileIngestStartTime;
1117 private final long processedFiles;
1118 private final long estimatedFilesToProcess;
1119 private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot;
1120 private final boolean jobCancelled;
1121 private final IngestJob.CancellationReason jobCancellationReason;
1122 private final List<String> cancelledDataSourceModules;
1128 Snapshot(
boolean getIngestTasksSnapshot) {
1129 this.dataSource = DataSourceIngestJob.this.dataSource.getName();
1130 this.jobId = DataSourceIngestJob.this.id;
1131 this.jobStartTime = DataSourceIngestJob.this.createTime;
1132 this.dataSourceLevelIngestModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule();
1139 for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) {
1140 if (pipeline.isRunning()) {
1141 this.fileIngestRunning =
true;
1143 Date pipelineStartTime = pipeline.getStartTime();
1144 if (null != pipelineStartTime && (null == this.fileIngestStartTime || pipelineStartTime.before(
this.fileIngestStartTime))) {
1145 this.fileIngestStartTime = pipelineStartTime;
1149 this.jobCancelled = cancelled;
1150 this.jobCancellationReason = cancellationReason;
1151 this.cancelledDataSourceModules =
new ArrayList<>(DataSourceIngestJob.this.cancelledDataSourceIngestModules);
1153 if (getIngestTasksSnapshot) {
1154 synchronized (DataSourceIngestJob.this.fileIngestProgressLock) {
1155 this.processedFiles = DataSourceIngestJob.this.processedFiles;
1156 this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess;
1157 this.snapShotTime =
new Date().getTime();
1159 this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId);
1162 this.processedFiles = 0;
1163 this.estimatedFilesToProcess = 0;
1164 this.snapShotTime =
new Date().getTime();
1165 this.tasksSnapshot = null;
1175 long getSnapshotTime() {
1176 return snapShotTime;
1185 String getDataSource() {
1205 long getJobStartTime() {
1206 return jobStartTime;
1209 DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() {
1210 return this.dataSourceLevelIngestModule;
1213 boolean fileIngestIsRunning() {
1214 return this.fileIngestRunning;
1217 Date fileIngestStartTime() {
1218 return this.fileIngestStartTime;
1228 return (
double) processedFiles / ((snapShotTime - jobStartTime) / 1000);
1236 long getFilesProcessed() {
1237 return processedFiles;
1246 long getFilesEstimated() {
1247 return estimatedFilesToProcess;
1250 long getRootQueueSize() {
1251 if (null == this.tasksSnapshot) {
1254 return this.tasksSnapshot.getRootQueueSize();
1257 long getDirQueueSize() {
1258 if (null == this.tasksSnapshot) {
1261 return this.tasksSnapshot.getDirectoryTasksQueueSize();
1264 long getFileQueueSize() {
1265 if (null == this.tasksSnapshot) {
1268 return this.tasksSnapshot.getFileQueueSize();
1271 long getDsQueueSize() {
1272 if (null == this.tasksSnapshot) {
1275 return this.tasksSnapshot.getDsQueueSize();
1278 long getRunningListSize() {
1279 if (null == this.tasksSnapshot) {
1282 return this.tasksSnapshot.getRunningListSize();
1285 boolean isCancelled() {
1286 return this.jobCancelled;
1294 IngestJob.CancellationReason getCancellationReason() {
1295 return this.jobCancellationReason;
1305 List<String> getCancelledDataSourceIngestModules() {
1306 return Collections.unmodifiableList(this.cancelledDataSourceModules);