Autopsy  4.4.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
DataSourceIngestJob.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2016 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.Date;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.concurrent.CopyOnWriteArrayList;
28 import java.util.concurrent.LinkedBlockingQueue;
29 import java.util.concurrent.atomic.AtomicLong;
30 import java.util.logging.Level;
31 import javax.swing.JOptionPane;
32 import org.netbeans.api.progress.ProgressHandle;
33 import org.openide.util.Cancellable;
34 import org.openide.util.NbBundle;
38 import org.sleuthkit.datamodel.AbstractFile;
39 import org.sleuthkit.datamodel.Content;
40 import org.sleuthkit.datamodel.IngestJobInfo;
41 import org.sleuthkit.datamodel.IngestJobInfo.IngestJobStatusType;
42 import org.sleuthkit.datamodel.IngestModuleInfo;
43 import org.sleuthkit.datamodel.IngestModuleInfo.IngestModuleType;
44 import org.sleuthkit.datamodel.SleuthkitCase;
45 import org.sleuthkit.datamodel.TskCoreException;
47 
52 final class DataSourceIngestJob {
53 
54  private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());
55 
60  private final IngestJob parentJob;
61  private static final AtomicLong nextJobId = new AtomicLong(0L);
62  private final long id;
63  private final IngestJobSettings settings;
64  private final Content dataSource;
65 
69  private static enum Stages {
70 
88  FINALIZATION
89  };
90  private volatile Stages stage = DataSourceIngestJob.Stages.INITIALIZATION;
91  private final Object stageCompletionCheckLock = new Object();
92 
101  private final Object dataSourceIngestPipelineLock = new Object();
102  private DataSourceIngestPipeline firstStageDataSourceIngestPipeline;
103  private DataSourceIngestPipeline secondStageDataSourceIngestPipeline;
104  private DataSourceIngestPipeline currentDataSourceIngestPipeline;
105 
113  private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelinesQueue = new LinkedBlockingQueue<>();
114  private final List<FileIngestPipeline> fileIngestPipelines = new ArrayList<>();
115 
127  private volatile boolean currentDataSourceIngestModuleCancelled;
128  private volatile boolean cancelled;
129  private volatile IngestJob.CancellationReason cancellationReason = IngestJob.CancellationReason.NOT_CANCELLED;
130  private final Object cancellationStateMonitor = new Object();
131  private final List<String> cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>();
132 
137  private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance();
138 
143  private final boolean doUI;
144 
149  private final Object dataSourceIngestProgressLock = new Object();
150  private ProgressHandle dataSourceIngestProgress;
151 
156  private final Object fileIngestProgressLock = new Object();
157  private final List<String> filesInProgress = new ArrayList<>();
158  private long estimatedFilesToProcess;
159  private long processedFiles;
160  private ProgressHandle fileIngestProgress;
161  private String currentFileIngestModule = "";
162  private String currentFileIngestTask = "";
163  private final List<IngestModuleInfo> ingestModules = new ArrayList<>();
164  private IngestJobInfo ingestJob;
165 
169  private final long createTime;
170 
182  DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean runInteractively) {
183  this.parentJob = parentJob;
184  this.id = DataSourceIngestJob.nextJobId.getAndIncrement();
185  this.dataSource = dataSource;
186  this.settings = settings;
187  this.doUI = runInteractively;
188  this.createTime = new Date().getTime();
189  this.createIngestPipelines();
190  }
191 
195  private void createIngestPipelines() {
196  List<IngestModuleTemplate> ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates();
197 
201  Map<String, IngestModuleTemplate> dataSourceModuleTemplates = new HashMap<>();
202  Map<String, IngestModuleTemplate> fileModuleTemplates = new HashMap<>();
203  for (IngestModuleTemplate template : ingestModuleTemplates) {
204  if (template.isDataSourceIngestModuleTemplate()) {
205  dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
206  }
207  if (template.isFileIngestModuleTemplate()) {
208  fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
209  }
210  }
211 
216  IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
217  List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
218  List<IngestModuleTemplate> fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
219  List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
220 
226  for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) {
227  firstStageDataSourceModuleTemplates.add(template);
228  }
229  for (IngestModuleTemplate template : fileModuleTemplates.values()) {
230  fileIngestModuleTemplates.add(template);
231  }
232 
236  this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates);
237  this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates);
238 
242  try {
243  int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads();
244  for (int i = 0; i < numberOfFileIngestThreads; ++i) {
245  FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates);
246  this.fileIngestPipelinesQueue.put(pipeline);
247  this.fileIngestPipelines.add(pipeline);
248  }
249  } catch (InterruptedException ex) {
255  Thread.currentThread().interrupt();
256  }
257  SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase();
258  try {
259  this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
260  this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase);
261  this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
262  } catch (TskCoreException ex) {
263  logger.log(Level.SEVERE, "Failed to add ingest modules to database.", ex);
264  }
265  }
266 
267  private void addIngestModules(List<IngestModuleTemplate> templates, IngestModuleType type, SleuthkitCase skCase) throws TskCoreException {
268  for (IngestModuleTemplate module : templates) {
269  ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber()));
270  }
271  }
272 
288  private static List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(Map<String, IngestModuleTemplate> ingestModuleTemplates, List<String> pipelineConfig) {
289  List<IngestModuleTemplate> templates = new ArrayList<>();
290  for (String moduleClassName : pipelineConfig) {
291  if (ingestModuleTemplates.containsKey(moduleClassName)) {
292  templates.add(ingestModuleTemplates.remove(moduleClassName));
293  }
294  }
295  return templates;
296  }
297 
303  long getId() {
304  return this.id;
305  }
306 
312  String getExecutionContext() {
313  return this.settings.getExecutionContext();
314  }
315 
321  Content getDataSource() {
322  return this.dataSource;
323  }
324 
331  boolean shouldProcessUnallocatedSpace() {
332  return this.settings.getProcessUnallocatedSpace();
333  }
334 
340  FilesSet getFileIngestFilter() {
341  return this.settings.getFileIngestFilter();
342  }
343 
349  boolean hasIngestPipeline() {
350  return this.hasFirstStageDataSourceIngestPipeline()
351  || this.hasFileIngestPipeline()
352  || this.hasSecondStageDataSourceIngestPipeline();
353  }
354 
361  private boolean hasFirstStageDataSourceIngestPipeline() {
362  return (this.firstStageDataSourceIngestPipeline.isEmpty() == false);
363  }
364 
371  private boolean hasSecondStageDataSourceIngestPipeline() {
372  return (this.secondStageDataSourceIngestPipeline.isEmpty() == false);
373  }
374 
380  private boolean hasFileIngestPipeline() {
381  if (!this.fileIngestPipelines.isEmpty()) {
382  return !this.fileIngestPipelines.get(0).isEmpty();
383  }
384  return false;
385  }
386 
392  List<IngestModuleError> start() {
393  List<IngestModuleError> errors = startUpIngestPipelines();
394  if (errors.isEmpty()) {
395  if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) {
396  logger.log(Level.INFO, "Starting first stage analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
397  this.startFirstStage();
398  } else if (this.hasSecondStageDataSourceIngestPipeline()) {
399  logger.log(Level.INFO, "Starting second stage analysis for {0} (jobId={1}), no first stage configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
400  this.startSecondStage();
401  }
402  try {
403  this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, "");
404  } catch (TskCoreException ex) {
405  logger.log(Level.SEVERE, "Failed to add ingest job to database.", ex);
406  }
407  }
408  return errors;
409  }
410 
417  private List<IngestModuleError> startUpIngestPipelines() {
418  List<IngestModuleError> errors = new ArrayList<>();
419 
420  /*
421  * Start the data-source-level ingest module pipelines.
422  */
423  errors.addAll(this.firstStageDataSourceIngestPipeline.startUp());
424  errors.addAll(this.secondStageDataSourceIngestPipeline.startUp());
425 
426  /*
427  * If the data-source-level ingest pipelines were successfully started,
428  * start the Start the file-level ingest pipelines (one per file ingest
429  * thread).
430  */
431  if (errors.isEmpty()) {
432  for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) {
433  errors.addAll(pipeline.startUp());
434  if (!errors.isEmpty()) {
435  /*
436  * If there are start up errors, the ingest job will not
437  * proceed, so shut down any file ingest pipelines that did
438  * start up.
439  */
440  while (!this.fileIngestPipelinesQueue.isEmpty()) {
441  FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll();
442  if (startedPipeline.isRunning()) {
443  List<IngestModuleError> shutDownErrors = startedPipeline.shutDown();
444  if (!shutDownErrors.isEmpty()) {
445  /*
446  * The start up errors will ultimately be
447  * reported to the user for possible remedy, but
448  * the shut down errors are logged here.
449  */
450  logIngestModuleErrors(shutDownErrors);
451  }
452  }
453  }
454  break;
455  }
456  }
457  }
458 
459  return errors;
460  }
461 
465  private void startFirstStage() {
466  this.stage = DataSourceIngestJob.Stages.FIRST;
467 
468  if (this.hasFileIngestPipeline()) {
469  synchronized (this.fileIngestProgressLock) {
470  this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor());
471  }
472  }
473 
474  if (this.doUI) {
478  if (this.hasFirstStageDataSourceIngestPipeline()) {
479  this.startDataSourceIngestProgressBar();
480  }
481  if (this.hasFileIngestPipeline()) {
482  this.startFileIngestProgressBar();
483  }
484  }
485 
490  synchronized (this.dataSourceIngestPipelineLock) {
491  this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline;
492  }
493 
497  if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) {
498  logger.log(Level.INFO, "Scheduling first stage data source and file level analysis tasks for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
499  DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this);
500  } else if (this.hasFirstStageDataSourceIngestPipeline()) {
501  logger.log(Level.INFO, "Scheduling first stage data source level analysis tasks for {0} (jobId={1}), no file level analysis configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
502  DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this);
503  } else {
504  logger.log(Level.INFO, "Scheduling file level analysis tasks for {0} (jobId={1}), no first stage data source level analysis configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
505  DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this);
506 
515  this.checkForStageCompleted();
516  }
517  }
518 
522  private void startSecondStage() {
523  logger.log(Level.INFO, "Starting second stage analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
524  this.stage = DataSourceIngestJob.Stages.SECOND;
525  if (this.doUI) {
526  this.startDataSourceIngestProgressBar();
527  }
528  synchronized (this.dataSourceIngestPipelineLock) {
529  this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline;
530  }
531  logger.log(Level.INFO, "Scheduling second stage data source level analysis tasks for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
532  DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this);
533  }
534 
538  private void startDataSourceIngestProgressBar() {
539  if (this.doUI) {
540  synchronized (this.dataSourceIngestProgressLock) {
541  String displayName = NbBundle.getMessage(this.getClass(),
542  "IngestJob.progress.dataSourceIngest.initialDisplayName",
543  this.dataSource.getName());
544  this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() {
545  @Override
546  public boolean cancel() {
547  // If this method is called, the user has already pressed
548  // the cancel button on the progress bar and the OK button
549  // of a cancelation confirmation dialog supplied by
550  // NetBeans. What remains to be done is to find out whether
551  // the user wants to cancel only the currently executing
552  // data source ingest module or the entire ingest job.
553  DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel();
554  String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title");
555  JOptionPane.showConfirmDialog(null, panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE);
556  if (panel.cancelAllDataSourceIngestModules()) {
557  DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
558  } else {
559  DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule();
560  }
561  return true;
562  }
563  });
564  this.dataSourceIngestProgress.start();
565  this.dataSourceIngestProgress.switchToIndeterminate();
566  }
567  }
568  }
569 
573  private void startFileIngestProgressBar() {
574  if (this.doUI) {
575  synchronized (this.fileIngestProgressLock) {
576  String displayName = NbBundle.getMessage(this.getClass(),
577  "IngestJob.progress.fileIngest.displayName",
578  this.dataSource.getName());
579  this.fileIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() {
580  @Override
581  public boolean cancel() {
582  // If this method is called, the user has already pressed
583  // the cancel button on the progress bar and the OK button
584  // of a cancelation confirmation dialog supplied by
585  // NetBeans.
586  DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
587  return true;
588  }
589  });
590  this.fileIngestProgress.start();
591  this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess);
592  }
593  }
594  }
595 
600  private void checkForStageCompleted() {
601  synchronized (this.stageCompletionCheckLock) {
602  if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(this)) {
603  switch (this.stage) {
604  case FIRST:
605  this.finishFirstStage();
606  break;
607  case SECOND:
608  this.finish();
609  break;
610  }
611  }
612  }
613  }
614 
619  private void finishFirstStage() {
620  logger.log(Level.INFO, "Finished first stage analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
621 
622  // Shut down the file ingest pipelines. Note that no shut down is
623  // required for the data source ingest pipeline because data source
624  // ingest modules do not have a shutdown() method.
625  List<IngestModuleError> errors = new ArrayList<>();
626  while (!this.fileIngestPipelinesQueue.isEmpty()) {
627  FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll();
628  if (pipeline.isRunning()) {
629  errors.addAll(pipeline.shutDown());
630  }
631  }
632  if (!errors.isEmpty()) {
633  logIngestModuleErrors(errors);
634  }
635 
636  if (this.doUI) {
637  // Finish the first stage data source ingest progress bar, if it hasn't
638  // already been finished.
639  synchronized (this.dataSourceIngestProgressLock) {
640  if (this.dataSourceIngestProgress != null) {
641  this.dataSourceIngestProgress.finish();
642  this.dataSourceIngestProgress = null;
643  }
644  }
645 
646  // Finish the file ingest progress bar, if it hasn't already
647  // been finished.
648  synchronized (this.fileIngestProgressLock) {
649  if (this.fileIngestProgress != null) {
650  this.fileIngestProgress.finish();
651  this.fileIngestProgress = null;
652  }
653  }
654  }
655 
659  if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) {
660  this.startSecondStage();
661  } else {
662  this.finish();
663  }
664  }
665 
669  private void finish() {
670  logger.log(Level.INFO, "Finished analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
671  this.stage = DataSourceIngestJob.Stages.FINALIZATION;
672 
673  if (this.doUI) {
674  // Finish the second stage data source ingest progress bar, if it hasn't
675  // already been finished.
676  synchronized (this.dataSourceIngestProgressLock) {
677  if (this.dataSourceIngestProgress != null) {
678  this.dataSourceIngestProgress.finish();
679  this.dataSourceIngestProgress = null;
680  }
681  }
682  }
683  if (this.cancelled) {
684  try {
685  ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED);
686  } catch (TskCoreException ex) {
687  logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex);
688  }
689  } else {
690  try {
691  ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED);
692  } catch (TskCoreException ex) {
693  logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex);
694  }
695  }
696  try {
697  this.ingestJob.setEndDateTime(new Date());
698  } catch (TskCoreException ex) {
699  logger.log(Level.SEVERE, "Failed to set end date for ingest job in database.", ex);
700  }
701  this.parentJob.dataSourceJobFinished(this);
702 
703  }
704 
711  void process(DataSourceIngestTask task) {
712  try {
713  synchronized (this.dataSourceIngestPipelineLock) {
714  if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
715  List<IngestModuleError> errors = new ArrayList<>();
716  errors.addAll(this.currentDataSourceIngestPipeline.process(task));
717  if (!errors.isEmpty()) {
718  logIngestModuleErrors(errors);
719  }
720  }
721  }
722 
723  if (this.doUI) {
728  synchronized (this.dataSourceIngestProgressLock) {
729  if (null != this.dataSourceIngestProgress) {
730  this.dataSourceIngestProgress.finish();
731  this.dataSourceIngestProgress = null;
732  }
733  }
734  }
735 
736  } finally {
737  DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
738  this.checkForStageCompleted();
739  }
740  }
741 
753  void process(FileIngestTask task) throws InterruptedException {
754  try {
755  if (!this.isCancelled()) {
756  FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take();
757  if (!pipeline.isEmpty()) {
758  AbstractFile file = task.getFile();
759 
760  synchronized (this.fileIngestProgressLock) {
761  ++this.processedFiles;
762  if (this.doUI) {
766  if (this.processedFiles <= this.estimatedFilesToProcess) {
767  this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles);
768  } else {
769  this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess);
770  }
771  this.filesInProgress.add(file.getName());
772  }
773  }
774 
778  List<IngestModuleError> errors = new ArrayList<>();
779  errors.addAll(pipeline.process(task));
780  if (!errors.isEmpty()) {
781  logIngestModuleErrors(errors);
782  }
783 
784  if (this.doUI && !this.cancelled) {
785  synchronized (this.fileIngestProgressLock) {
790  this.filesInProgress.remove(file.getName());
791  if (this.filesInProgress.size() > 0) {
792  this.fileIngestProgress.progress(this.filesInProgress.get(0));
793  } else {
794  this.fileIngestProgress.progress("");
795  }
796  }
797  }
798  }
799  this.fileIngestPipelinesQueue.put(pipeline);
800  }
801  } finally {
802  DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
803  this.checkForStageCompleted();
804  }
805  }
806 
814  void addFiles(List<AbstractFile> files) {
815  if (DataSourceIngestJob.Stages.FIRST == this.stage) {
816  for (AbstractFile file : files) {
817  DataSourceIngestJob.taskScheduler.scheduleFileIngestTask(this, file);
818  }
819  } else {
820  DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS
821  }
822 
829  this.checkForStageCompleted();
830  }
831 
838  void updateDataSourceIngestProgressBarDisplayName(String displayName) {
839  if (this.doUI && !this.cancelled) {
840  synchronized (this.dataSourceIngestProgressLock) {
841  this.dataSourceIngestProgress.setDisplayName(displayName);
842  }
843  }
844  }
845 
854  void switchDataSourceIngestProgressBarToDeterminate(int workUnits) {
855  if (this.doUI && !this.cancelled) {
856  synchronized (this.dataSourceIngestProgressLock) {
857  if (null != this.dataSourceIngestProgress) {
858  this.dataSourceIngestProgress.switchToDeterminate(workUnits);
859  }
860  }
861  }
862  }
863 
869  void switchDataSourceIngestProgressBarToIndeterminate() {
870  if (this.doUI && !this.cancelled) {
871  synchronized (this.dataSourceIngestProgressLock) {
872  if (null != this.dataSourceIngestProgress) {
873  this.dataSourceIngestProgress.switchToIndeterminate();
874  }
875  }
876  }
877  }
878 
885  void advanceDataSourceIngestProgressBar(int workUnits) {
886  if (this.doUI && !this.cancelled) {
887  synchronized (this.dataSourceIngestProgressLock) {
888  if (null != this.dataSourceIngestProgress) {
889  this.dataSourceIngestProgress.progress("", workUnits);
890  }
891  }
892  }
893  }
894 
901  void advanceDataSourceIngestProgressBar(String currentTask) {
902  if (this.doUI && !this.cancelled) {
903  synchronized (this.dataSourceIngestProgressLock) {
904  if (null != this.dataSourceIngestProgress) {
905  this.dataSourceIngestProgress.progress(currentTask);
906  }
907  }
908  }
909  }
910 
919  void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) {
920  if (this.doUI && !this.cancelled) {
921  synchronized (this.fileIngestProgressLock) {
922  this.dataSourceIngestProgress.progress(currentTask, workUnits);
923  }
924  }
925  }
926 
934  boolean currentDataSourceIngestModuleIsCancelled() {
935  return this.currentDataSourceIngestModuleCancelled;
936  }
937 
944  void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) {
945  this.currentDataSourceIngestModuleCancelled = false;
946  this.cancelledDataSourceIngestModules.add(moduleDisplayName);
947 
948  if (this.doUI) {
956  synchronized (this.dataSourceIngestProgressLock) {
957  this.dataSourceIngestProgress.finish();
958  this.dataSourceIngestProgress = null;
959  this.startDataSourceIngestProgressBar();
960  }
961  }
962  }
963 
969  DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() {
970  if (null != this.currentDataSourceIngestPipeline) {
971  return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule();
972  } else {
973  return null;
974  }
975  }
976 
981  void cancelCurrentDataSourceIngestModule() {
982  this.currentDataSourceIngestModuleCancelled = true;
983  }
984 
991  void cancel(IngestJob.CancellationReason reason) {
992  if (this.doUI) {
997  synchronized (this.dataSourceIngestProgressLock) {
998  if (dataSourceIngestProgress != null) {
999  final String displayName = NbBundle.getMessage(this.getClass(),
1000  "IngestJob.progress.dataSourceIngest.initialDisplayName",
1001  dataSource.getName());
1002  dataSourceIngestProgress.setDisplayName(
1003  NbBundle.getMessage(this.getClass(),
1004  "IngestJob.progress.cancelling",
1005  displayName));
1006  }
1007  }
1008 
1013  synchronized (this.fileIngestProgressLock) {
1014  if (this.fileIngestProgress != null) {
1015  final String displayName = NbBundle.getMessage(this.getClass(),
1016  "IngestJob.progress.fileIngest.displayName",
1017  this.dataSource.getName());
1018  this.fileIngestProgress.setDisplayName(
1019  NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling",
1020  displayName));
1021  if (!this.currentFileIngestModule.isEmpty() && !this.currentFileIngestTask.isEmpty()) {
1022  this.fileIngestProgress.progress(NbBundle.getMessage(this.getClass(),
1023  "IngestJob.progress.fileIngest.cancelMessage",
1024  this.currentFileIngestModule, this.currentFileIngestTask));
1025  }
1026 
1027  }
1028  }
1029  }
1030 
1031  /*
1032  * If the work is not already done, show this job as cancelled for the
1033  * given reason.
1034  */
1035  if (Stages.FINALIZATION != stage) {
1036  synchronized (cancellationStateMonitor) {
1037  /*
1038  * These fields are volatile for reading, synchronized on the
1039  * monitor here for writing.
1040  */
1041  this.cancelled = true;
1042  this.cancellationReason = reason;
1043  }
1044  }
1045 
1050  DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this);
1051  this.checkForStageCompleted();
1052  }
1053 
1061  void setCurrentFileIngestModule(String moduleName, String taskName) {
1062  this.currentFileIngestModule = moduleName;
1063  this.currentFileIngestTask = taskName;
1064  }
1065 
1072  boolean isCancelled() {
1073  return this.cancelled;
1074  }
1075 
1081  IngestJob.CancellationReason getCancellationReason() {
1082  return this.cancellationReason;
1083  }
1084 
1090  private void logIngestModuleErrors(List<IngestModuleError> errors) {
1091  for (IngestModuleError error : errors) {
1092  DataSourceIngestJob.logger.log(Level.SEVERE, String.format("%s experienced an error analyzing %s (jobId=%d)", error.getModuleDisplayName(), dataSource.getName(), this.id), error.getThrowable()); //NON-NLS
1093  }
1094  }
1095 
1101  Snapshot getSnapshot(boolean getIngestTasksSnapshot) {
1102  return new Snapshot(getIngestTasksSnapshot);
1103  }
1104 
1108  final class Snapshot {
1109 
1110  private final String dataSource;
1111  private final long jobId;
1112  private final long jobStartTime;
1113  private final long snapShotTime;
1114  private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule;
1115  private boolean fileIngestRunning;
1116  private Date fileIngestStartTime;
1117  private final long processedFiles;
1118  private final long estimatedFilesToProcess;
1119  private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot;
1120  private final boolean jobCancelled;
1121  private final IngestJob.CancellationReason jobCancellationReason;
1122  private final List<String> cancelledDataSourceModules;
1123 
1128  Snapshot(boolean getIngestTasksSnapshot) {
1129  this.dataSource = DataSourceIngestJob.this.dataSource.getName();
1130  this.jobId = DataSourceIngestJob.this.id;
1131  this.jobStartTime = DataSourceIngestJob.this.createTime;
1132  this.dataSourceLevelIngestModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule();
1133 
1139  for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) {
1140  if (pipeline.isRunning()) {
1141  this.fileIngestRunning = true;
1142  }
1143  Date pipelineStartTime = pipeline.getStartTime();
1144  if (null != pipelineStartTime && (null == this.fileIngestStartTime || pipelineStartTime.before(this.fileIngestStartTime))) {
1145  this.fileIngestStartTime = pipelineStartTime;
1146  }
1147  }
1148 
1149  this.jobCancelled = cancelled;
1150  this.jobCancellationReason = cancellationReason;
1151  this.cancelledDataSourceModules = new ArrayList<>(DataSourceIngestJob.this.cancelledDataSourceIngestModules);
1152 
1153  if (getIngestTasksSnapshot) {
1154  synchronized (DataSourceIngestJob.this.fileIngestProgressLock) {
1155  this.processedFiles = DataSourceIngestJob.this.processedFiles;
1156  this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess;
1157  this.snapShotTime = new Date().getTime();
1158  }
1159  this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId);
1160 
1161  } else {
1162  this.processedFiles = 0;
1163  this.estimatedFilesToProcess = 0;
1164  this.snapShotTime = new Date().getTime();
1165  this.tasksSnapshot = null;
1166  }
1167  }
1168 
1175  long getSnapshotTime() {
1176  return snapShotTime;
1177  }
1178 
1185  String getDataSource() {
1186  return dataSource;
1187  }
1188 
1195  long getJobId() {
1196  return this.jobId;
1197  }
1198 
1205  long getJobStartTime() {
1206  return jobStartTime;
1207  }
1208 
1209  DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() {
1210  return this.dataSourceLevelIngestModule;
1211  }
1212 
1213  boolean fileIngestIsRunning() {
1214  return this.fileIngestRunning;
1215  }
1216 
1217  Date fileIngestStartTime() {
1218  return this.fileIngestStartTime;
1219  }
1220 
1227  double getSpeed() {
1228  return (double) processedFiles / ((snapShotTime - jobStartTime) / 1000);
1229  }
1230 
1236  long getFilesProcessed() {
1237  return processedFiles;
1238  }
1239 
1246  long getFilesEstimated() {
1247  return estimatedFilesToProcess;
1248  }
1249 
1250  long getRootQueueSize() {
1251  if (null == this.tasksSnapshot) {
1252  return 0;
1253  }
1254  return this.tasksSnapshot.getRootQueueSize();
1255  }
1256 
1257  long getDirQueueSize() {
1258  if (null == this.tasksSnapshot) {
1259  return 0;
1260  }
1261  return this.tasksSnapshot.getDirectoryTasksQueueSize();
1262  }
1263 
1264  long getFileQueueSize() {
1265  if (null == this.tasksSnapshot) {
1266  return 0;
1267  }
1268  return this.tasksSnapshot.getFileQueueSize();
1269  }
1270 
1271  long getDsQueueSize() {
1272  if (null == this.tasksSnapshot) {
1273  return 0;
1274  }
1275  return this.tasksSnapshot.getDsQueueSize();
1276  }
1277 
1278  long getRunningListSize() {
1279  if (null == this.tasksSnapshot) {
1280  return 0;
1281  }
1282  return this.tasksSnapshot.getRunningListSize();
1283  }
1284 
1285  boolean isCancelled() {
1286  return this.jobCancelled;
1287  }
1288 
1294  IngestJob.CancellationReason getCancellationReason() {
1295  return this.jobCancellationReason;
1296  }
1297 
1305  List<String> getCancelledDataSourceIngestModules() {
1306  return Collections.unmodifiableList(this.cancelledDataSourceModules);
1307  }
1308 
1309  }
1310 
1311 }

Copyright © 2012-2016 Basis Technology. Generated on: Fri Sep 29 2017
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.