Autopsy  4.9.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
DataSourceIngestJob.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.io.Serializable;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.Date;
25 import java.util.LinkedHashMap;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.concurrent.CopyOnWriteArrayList;
29 import java.util.concurrent.LinkedBlockingQueue;
30 import java.util.concurrent.atomic.AtomicLong;
31 import java.util.logging.Level;
32 import javax.swing.JOptionPane;
33 import org.netbeans.api.progress.ProgressHandle;
34 import org.openide.util.Cancellable;
35 import org.openide.util.NbBundle;
36 import org.openide.windows.WindowManager;
44 import org.sleuthkit.datamodel.AbstractFile;
45 import org.sleuthkit.datamodel.Content;
46 import org.sleuthkit.datamodel.IngestJobInfo;
47 import org.sleuthkit.datamodel.IngestJobInfo.IngestJobStatusType;
48 import org.sleuthkit.datamodel.IngestModuleInfo;
49 import org.sleuthkit.datamodel.IngestModuleInfo.IngestModuleType;
50 import org.sleuthkit.datamodel.SleuthkitCase;
51 import org.sleuthkit.datamodel.TskCoreException;
53 
58 public final class DataSourceIngestJob {
59 
60  private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());
61 
68  private final IngestJob parentJob;
69  private static final AtomicLong nextJobId = new AtomicLong(0L);
70  private final long id;
71  private final IngestJobSettings settings;
72  private final Content dataSource;
73  private final List<AbstractFile> files = new ArrayList<>();
74 
78  private static enum Stages {
79 
97  FINALIZATION
98  };
100  private final Object stageCompletionCheckLock = new Object();
101 
110  private final Object dataSourceIngestPipelineLock = new Object();
111  private DataSourceIngestPipeline firstStageDataSourceIngestPipeline;
112  private DataSourceIngestPipeline secondStageDataSourceIngestPipeline;
113  private DataSourceIngestPipeline currentDataSourceIngestPipeline;
114 
122  private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelinesQueue = new LinkedBlockingQueue<>();
123  private final List<FileIngestPipeline> fileIngestPipelines = new ArrayList<>();
124 
136  private volatile boolean currentDataSourceIngestModuleCancelled;
137  private final List<String> cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>();
138  private volatile boolean cancelled;
140 
145  private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance();
146 
151  private final boolean doUI;
152 
157  private final Object dataSourceIngestProgressLock = new Object();
158  private ProgressHandle dataSourceIngestProgress;
159 
164  private final Object fileIngestProgressLock = new Object();
165  private final List<String> filesInProgress = new ArrayList<>();
167  private long processedFiles;
168  private ProgressHandle fileIngestProgress;
169  private String currentFileIngestModule = "";
170  private String currentFileIngestTask = "";
171  private final List<IngestModuleInfo> ingestModules = new ArrayList<>();
172  private volatile IngestJobInfo ingestJob;
173 
177  private final long createTime;
178 
190  DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean runInteractively) {
191  this(parentJob, dataSource, Collections.emptyList(), settings, runInteractively);
192  }
193 
207  DataSourceIngestJob(IngestJob parentJob, Content dataSource, List<AbstractFile> files, IngestJobSettings settings, boolean runInteractively) {
208  this.parentJob = parentJob;
209  this.id = DataSourceIngestJob.nextJobId.getAndIncrement();
210  this.dataSource = dataSource;
211  this.files.addAll(files);
212  this.settings = settings;
213  this.doUI = runInteractively;
214  this.createTime = new Date().getTime();
215  this.createIngestPipelines();
216  }
217 
221  private void createIngestPipelines() {
222  List<IngestModuleTemplate> ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates();
223 
227  Map<String, IngestModuleTemplate> dataSourceModuleTemplates = new LinkedHashMap<>();
228  Map<String, IngestModuleTemplate> fileModuleTemplates = new LinkedHashMap<>();
229  for (IngestModuleTemplate template : ingestModuleTemplates) {
230  if (template.isDataSourceIngestModuleTemplate()) {
231  dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
232  }
233  if (template.isFileIngestModuleTemplate()) {
234  fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
235  }
236  }
237 
242  IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
243  List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
244  List<IngestModuleTemplate> fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
245  List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
246 
252  for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) {
253  firstStageDataSourceModuleTemplates.add(template);
254  }
255  for (IngestModuleTemplate template : fileModuleTemplates.values()) {
256  fileIngestModuleTemplates.add(template);
257  }
258 
262  this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates);
263  this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates);
264 
268  try {
269  int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads();
270  for (int i = 0; i < numberOfFileIngestThreads; ++i) {
271  FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates);
272  this.fileIngestPipelinesQueue.put(pipeline);
273  this.fileIngestPipelines.add(pipeline);
274  }
275  } catch (InterruptedException ex) {
281  Thread.currentThread().interrupt();
282  }
283  try {
284  SleuthkitCase skCase = Case.getCurrentCaseThrows().getSleuthkitCase();
285  this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
286  this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase);
287  this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase);
288  } catch (TskCoreException | NoCurrentCaseException ex) {
289  logger.log(Level.SEVERE, "Failed to add ingest modules to database.", ex);
290  }
291  }
292 
293  private void addIngestModules(List<IngestModuleTemplate> templates, IngestModuleType type, SleuthkitCase skCase) throws TskCoreException {
294  for (IngestModuleTemplate module : templates) {
295  ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber()));
296  }
297  }
298 
314  private static List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(Map<String, IngestModuleTemplate> ingestModuleTemplates, List<String> pipelineConfig) {
315  List<IngestModuleTemplate> templates = new ArrayList<>();
316  for (String moduleClassName : pipelineConfig) {
317  if (ingestModuleTemplates.containsKey(moduleClassName)) {
318  templates.add(ingestModuleTemplates.remove(moduleClassName));
319  }
320  }
321  return templates;
322  }
323 
329  long getId() {
330  return this.id;
331  }
332 
338  String getExecutionContext() {
339  return this.settings.getExecutionContext();
340  }
341 
347  Content getDataSource() {
348  return this.dataSource;
349  }
350 
357  boolean shouldProcessUnallocatedSpace() {
358  return this.settings.getProcessUnallocatedSpace();
359  }
360 
366  FilesSet getFileIngestFilter() {
367  return this.settings.getFileFilter();
368  }
369 
375  boolean hasIngestPipeline() {
377  || this.hasFileIngestPipeline()
379  }
380 
388  return (this.firstStageDataSourceIngestPipeline.isEmpty() == false);
389  }
390 
398  return (this.secondStageDataSourceIngestPipeline.isEmpty() == false);
399  }
400 
406  private boolean hasFileIngestPipeline() {
407  if (!this.fileIngestPipelines.isEmpty()) {
408  return !this.fileIngestPipelines.get(0).isEmpty();
409  }
410  return false;
411  }
412 
418  List<IngestModuleError> start() {
419  List<IngestModuleError> errors = startUpIngestPipelines();
420  if (errors.isEmpty()) {
421  try {
422  this.ingestJob = Case.getCurrentCaseThrows().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, "");
423  } catch (TskCoreException | NoCurrentCaseException ex) {
424  logger.log(Level.SEVERE, "Failed to add ingest job to database.", ex);
425  }
427  logger.log(Level.INFO, "Starting first stage analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
428  this.startFirstStage();
429  } else if (this.hasSecondStageDataSourceIngestPipeline()) {
430  logger.log(Level.INFO, "Starting second stage analysis for {0} (jobId={1}), no first stage configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
431  this.startSecondStage();
432  }
433  }
434  return errors;
435  }
436 
443  private List<IngestModuleError> startUpIngestPipelines() {
444  List<IngestModuleError> errors = new ArrayList<>();
445 
446  /*
447  * Start the data-source-level ingest module pipelines.
448  */
449  errors.addAll(this.firstStageDataSourceIngestPipeline.startUp());
450  errors.addAll(this.secondStageDataSourceIngestPipeline.startUp());
451 
452  /*
453  * If the data-source-level ingest pipelines were successfully started,
454  * start the Start the file-level ingest pipelines (one per file ingest
455  * thread).
456  */
457  if (errors.isEmpty()) {
458  for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) {
459  errors.addAll(pipeline.startUp());
460  if (!errors.isEmpty()) {
461  /*
462  * If there are start up errors, the ingest job will not
463  * proceed, so shut down any file ingest pipelines that did
464  * start up.
465  */
466  while (!this.fileIngestPipelinesQueue.isEmpty()) {
467  FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll();
468  if (startedPipeline.isRunning()) {
469  List<IngestModuleError> shutDownErrors = startedPipeline.shutDown();
470  if (!shutDownErrors.isEmpty()) {
471  /*
472  * The start up errors will ultimately be
473  * reported to the user for possible remedy, but
474  * the shut down errors are logged here.
475  */
476  logIngestModuleErrors(shutDownErrors);
477  }
478  }
479  }
480  break;
481  }
482  }
483  }
484 
485  return errors;
486  }
487 
491  private void startFirstStage() {
492  this.stage = DataSourceIngestJob.Stages.FIRST;
493 
494  if (this.hasFileIngestPipeline()) {
495  synchronized (this.fileIngestProgressLock) {
496  this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor());
497  }
498  }
499 
500  if (this.doUI) {
506  }
507  if (this.hasFileIngestPipeline()) {
509  }
510  }
511 
516  synchronized (this.dataSourceIngestPipelineLock) {
517  this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline;
518  }
519 
524  logger.log(Level.INFO, "Scheduling first stage data source and file level analysis tasks for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
525  DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this);
526  } else if (this.hasFirstStageDataSourceIngestPipeline()) {
527  logger.log(Level.INFO, "Scheduling first stage data source level analysis tasks for {0} (jobId={1}), no file level analysis configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
528  DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this);
529  } else {
530  logger.log(Level.INFO, "Scheduling file level analysis tasks for {0} (jobId={1}), no first stage data source level analysis configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS
531  DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this, this.files);
532 
541  this.checkForStageCompleted();
542  }
543  }
544 
548  private void startSecondStage() {
549  logger.log(Level.INFO, "Starting second stage analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
550  this.stage = DataSourceIngestJob.Stages.SECOND;
551  if (this.doUI) {
553  }
554  synchronized (this.dataSourceIngestPipelineLock) {
555  this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline;
556  }
557  logger.log(Level.INFO, "Scheduling second stage data source level analysis tasks for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
558  DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this);
559  }
560 
565  if (this.doUI) {
566  synchronized (this.dataSourceIngestProgressLock) {
567  String displayName = NbBundle.getMessage(this.getClass(),
568  "IngestJob.progress.dataSourceIngest.initialDisplayName",
569  this.dataSource.getName());
570  this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() {
571  @Override
572  public boolean cancel() {
573  // If this method is called, the user has already pressed
574  // the cancel button on the progress bar and the OK button
575  // of a cancelation confirmation dialog supplied by
576  // NetBeans. What remains to be done is to find out whether
577  // the user wants to cancel only the currently executing
578  // data source ingest module or the entire ingest job.
579  DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel();
580  String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title");
581  JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE);
582  if (panel.cancelAllDataSourceIngestModules()) {
584  } else {
585  DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule();
586  }
587  return true;
588  }
589  });
590  this.dataSourceIngestProgress.start();
591  this.dataSourceIngestProgress.switchToIndeterminate();
592  }
593  }
594  }
595 
599  private void startFileIngestProgressBar() {
600  if (this.doUI) {
601  synchronized (this.fileIngestProgressLock) {
602  String displayName = NbBundle.getMessage(this.getClass(),
603  "IngestJob.progress.fileIngest.displayName",
604  this.dataSource.getName());
605  this.fileIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() {
606  @Override
607  public boolean cancel() {
608  // If this method is called, the user has already pressed
609  // the cancel button on the progress bar and the OK button
610  // of a cancelation confirmation dialog supplied by
611  // NetBeans.
613  return true;
614  }
615  });
616  this.fileIngestProgress.start();
617  this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess);
618  }
619  }
620  }
621 
626  private void checkForStageCompleted() {
627  synchronized (this.stageCompletionCheckLock) {
628  if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(this)) {
629  switch (this.stage) {
630  case FIRST:
631  this.finishFirstStage();
632  break;
633  case SECOND:
634  this.finish();
635  break;
636  }
637  }
638  }
639  }
640 
645  private void finishFirstStage() {
646  logger.log(Level.INFO, "Finished first stage analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
647 
648  // Shut down the file ingest pipelines. Note that no shut down is
649  // required for the data source ingest pipeline because data source
650  // ingest modules do not have a shutdown() method.
651  List<IngestModuleError> errors = new ArrayList<>();
652  while (!this.fileIngestPipelinesQueue.isEmpty()) {
653  FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll();
654  if (pipeline.isRunning()) {
655  errors.addAll(pipeline.shutDown());
656  }
657  }
658  if (!errors.isEmpty()) {
659  logIngestModuleErrors(errors);
660  }
661 
662  if (this.doUI) {
663  // Finish the first stage data source ingest progress bar, if it hasn't
664  // already been finished.
665  synchronized (this.dataSourceIngestProgressLock) {
666  if (this.dataSourceIngestProgress != null) {
667  this.dataSourceIngestProgress.finish();
668  this.dataSourceIngestProgress = null;
669  }
670  }
671 
672  // Finish the file ingest progress bar, if it hasn't already
673  // been finished.
674  synchronized (this.fileIngestProgressLock) {
675  if (this.fileIngestProgress != null) {
676  this.fileIngestProgress.finish();
677  this.fileIngestProgress = null;
678  }
679  }
680  }
681 
685  if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) {
686  this.startSecondStage();
687  } else {
688  this.finish();
689  }
690  }
691 
695  private void finish() {
696  logger.log(Level.INFO, "Finished analysis for {0} (jobId={1})", new Object[]{dataSource.getName(), this.id}); //NON-NLS
698 
699  if (this.doUI) {
700  // Finish the second stage data source ingest progress bar, if it hasn't
701  // already been finished.
702  synchronized (this.dataSourceIngestProgressLock) {
703  if (this.dataSourceIngestProgress != null) {
704  this.dataSourceIngestProgress.finish();
705  this.dataSourceIngestProgress = null;
706  }
707  }
708  }
709  if (ingestJob != null) {
710  if (this.cancelled) {
711  try {
712  ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED);
713  } catch (TskCoreException ex) {
714  logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex);
715  }
716  } else {
717  try {
718  ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED);
719  } catch (TskCoreException ex) {
720  logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex);
721  }
722  }
723  try {
724  this.ingestJob.setEndDateTime(new Date());
725  } catch (TskCoreException ex) {
726  logger.log(Level.SEVERE, "Failed to set end date for ingest job in database.", ex);
727  }
728  }
729  this.parentJob.dataSourceJobFinished(this);
730  }
731 
738  void process(DataSourceIngestTask task) {
739  try {
740  synchronized (this.dataSourceIngestPipelineLock) {
741  if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
742  List<IngestModuleError> errors = new ArrayList<>();
743  errors.addAll(this.currentDataSourceIngestPipeline.process(task));
744  if (!errors.isEmpty()) {
745  logIngestModuleErrors(errors);
746  }
747  }
748  }
749 
750  if (this.doUI) {
755  synchronized (this.dataSourceIngestProgressLock) {
756  if (null != this.dataSourceIngestProgress) {
757  this.dataSourceIngestProgress.finish();
758  this.dataSourceIngestProgress = null;
759  }
760  }
761  }
762 
763  } finally {
764  DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
765  this.checkForStageCompleted();
766  }
767  }
768 
780  void process(FileIngestTask task) throws InterruptedException {
781  try {
782  if (!this.isCancelled()) {
783  FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take();
784  if (!pipeline.isEmpty()) {
785  AbstractFile file = task.getFile();
786 
787  synchronized (this.fileIngestProgressLock) {
788  ++this.processedFiles;
789  if (this.doUI) {
793  if (this.processedFiles <= this.estimatedFilesToProcess) {
794  this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles);
795  } else {
796  this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess);
797  }
798  this.filesInProgress.add(file.getName());
799  }
800  }
801 
805  List<IngestModuleError> errors = new ArrayList<>();
806  errors.addAll(pipeline.process(task));
807  if (!errors.isEmpty()) {
808  logIngestModuleErrors(errors);
809  }
810 
811  if (this.doUI && !this.cancelled) {
812  synchronized (this.fileIngestProgressLock) {
817  this.filesInProgress.remove(file.getName());
818  if (this.filesInProgress.size() > 0) {
819  this.fileIngestProgress.progress(this.filesInProgress.get(0));
820  } else {
821  this.fileIngestProgress.progress("");
822  }
823  }
824  }
825  }
826  this.fileIngestPipelinesQueue.put(pipeline);
827  }
828  } finally {
829  DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task);
830  this.checkForStageCompleted();
831  }
832  }
833 
841  void addFiles(List<AbstractFile> files) {
842  if (DataSourceIngestJob.Stages.FIRST == this.stage) {
843  DataSourceIngestJob.taskScheduler.fastTrackFileIngestTasks(this, files);
844  } else {
845  DataSourceIngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS
846  }
847 
854  this.checkForStageCompleted();
855  }
856 
863  void updateDataSourceIngestProgressBarDisplayName(String displayName) {
864  if (this.doUI && !this.cancelled) {
865  synchronized (this.dataSourceIngestProgressLock) {
866  this.dataSourceIngestProgress.setDisplayName(displayName);
867  }
868  }
869  }
870 
879  void switchDataSourceIngestProgressBarToDeterminate(int workUnits) {
880  if (this.doUI && !this.cancelled) {
881  synchronized (this.dataSourceIngestProgressLock) {
882  if (null != this.dataSourceIngestProgress) {
883  this.dataSourceIngestProgress.switchToDeterminate(workUnits);
884  }
885  }
886  }
887  }
888 
894  void switchDataSourceIngestProgressBarToIndeterminate() {
895  if (this.doUI && !this.cancelled) {
896  synchronized (this.dataSourceIngestProgressLock) {
897  if (null != this.dataSourceIngestProgress) {
898  this.dataSourceIngestProgress.switchToIndeterminate();
899  }
900  }
901  }
902  }
903 
910  void advanceDataSourceIngestProgressBar(int workUnits) {
911  if (this.doUI && !this.cancelled) {
912  synchronized (this.dataSourceIngestProgressLock) {
913  if (null != this.dataSourceIngestProgress) {
914  this.dataSourceIngestProgress.progress("", workUnits);
915  }
916  }
917  }
918  }
919 
926  void advanceDataSourceIngestProgressBar(String currentTask) {
927  if (this.doUI && !this.cancelled) {
928  synchronized (this.dataSourceIngestProgressLock) {
929  if (null != this.dataSourceIngestProgress) {
930  this.dataSourceIngestProgress.progress(currentTask);
931  }
932  }
933  }
934  }
935 
944  void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) {
945  if (this.doUI && !this.cancelled) {
946  synchronized (this.fileIngestProgressLock) {
947  this.dataSourceIngestProgress.progress(currentTask, workUnits);
948  }
949  }
950  }
951 
959  boolean currentDataSourceIngestModuleIsCancelled() {
961  }
962 
969  void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) {
970  this.currentDataSourceIngestModuleCancelled = false;
971  this.cancelledDataSourceIngestModules.add(moduleDisplayName);
972 
973  if (this.doUI) {
981  synchronized (this.dataSourceIngestProgressLock) {
982  this.dataSourceIngestProgress.finish();
983  this.dataSourceIngestProgress = null;
985  }
986  }
987  }
988 
994  DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() {
995  if (null != this.currentDataSourceIngestPipeline) {
996  return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule();
997  } else {
998  return null;
999  }
1000  }
1001 
1006  void cancelCurrentDataSourceIngestModule() {
1007  this.currentDataSourceIngestModuleCancelled = true;
1008  }
1009 
1016  void cancel(IngestJob.CancellationReason reason) {
1017  this.cancelled = true;
1018  this.cancellationReason = reason;
1019  DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this);
1020 
1021  if (this.doUI) {
1022  synchronized (this.dataSourceIngestProgressLock) {
1023  if (null != dataSourceIngestProgress) {
1024  dataSourceIngestProgress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName()));
1025  dataSourceIngestProgress.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling"));
1026  }
1027  }
1028 
1029  synchronized (this.fileIngestProgressLock) {
1030  if (null != this.fileIngestProgress) {
1031  this.fileIngestProgress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()));
1032  this.fileIngestProgress.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling"));
1033  }
1034  }
1035  }
1036  }
1037 
1045  void setCurrentFileIngestModule(String moduleName, String taskName) {
1046  this.currentFileIngestModule = moduleName;
1047  this.currentFileIngestTask = taskName;
1048  }
1049 
1056  boolean isCancelled() {
1057  return this.cancelled;
1058  }
1059 
1065  IngestJob.CancellationReason getCancellationReason() {
1066  return this.cancellationReason;
1067  }
1068 
1074  private void logIngestModuleErrors(List<IngestModuleError> errors) {
1075  for (IngestModuleError error : errors) {
1076  DataSourceIngestJob.logger.log(Level.SEVERE, String.format("%s experienced an error analyzing %s (jobId=%d)", error.getModuleDisplayName(), dataSource.getName(), this.id), error.getThrowable()); //NON-NLS
1077  }
1078  }
1079 
1085  Snapshot getSnapshot(boolean getIngestTasksSnapshot) {
1091  boolean fileIngestRunning = false;
1092  Date fileIngestStartTime = null;
1093 
1094  for (FileIngestPipeline pipeline : this.fileIngestPipelines) {
1095  if (pipeline.isRunning()) {
1096  fileIngestRunning = true;
1097  }
1098  Date pipelineStartTime = pipeline.getStartTime();
1099  if (null != pipelineStartTime && (null == fileIngestStartTime || pipelineStartTime.before(fileIngestStartTime))) {
1100  fileIngestStartTime = pipelineStartTime;
1101  }
1102  }
1103 
1104  long processedFilesCount = 0;
1105  long estimatedFilesToProcessCount = 0;
1106  long snapShotTime = new Date().getTime();
1107  IngestJobTasksSnapshot tasksSnapshot = null;
1108 
1109  if (getIngestTasksSnapshot) {
1110  synchronized (fileIngestProgressLock) {
1111  processedFilesCount = this.processedFiles;
1112  estimatedFilesToProcessCount = this.estimatedFilesToProcess;
1113  snapShotTime = new Date().getTime();
1114  }
1115  tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(id);
1116 
1117  }
1118 
1119  return new Snapshot(this.dataSource.getName(), id, createTime,
1120  getCurrentDataSourceIngestModule(), fileIngestRunning, fileIngestStartTime,
1122  processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot);
1123  }
1124 
1128  public static final class Snapshot implements Serializable {
1129 
1130  private static final long serialVersionUID = 1L;
1131 
1132  private final String dataSource;
1133  private final long jobId;
1134  private final long jobStartTime;
1135  private final long snapShotTime;
1136  transient private final PipelineModule dataSourceLevelIngestModule;
1137  private final boolean fileIngestRunning;
1138  private final Date fileIngestStartTime;
1139  private final long processedFiles;
1140  private final long estimatedFilesToProcess;
1141  private final IngestJobTasksSnapshot tasksSnapshot;
1142  transient private final boolean jobCancelled;
1144  transient private final List<String> cancelledDataSourceModules;
1145 
1150  Snapshot(String dataSourceName, long jobId, long jobStartTime, PipelineModule dataSourceIngestModule,
1151  boolean fileIngestRunning, Date fileIngestStartTime,
1152  boolean jobCancelled, CancellationReason cancellationReason, List<String> cancelledModules,
1153  long processedFiles, long estimatedFilesToProcess,
1154  long snapshotTime, IngestJobTasksSnapshot tasksSnapshot) {
1155  this.dataSource = dataSourceName;
1156  this.jobId = jobId;
1157  this.jobStartTime = jobStartTime;
1158  this.dataSourceLevelIngestModule = dataSourceIngestModule;
1159 
1160  this.fileIngestRunning = fileIngestRunning;
1161  this.fileIngestStartTime = fileIngestStartTime;
1162  this.jobCancelled = jobCancelled;
1163  this.jobCancellationReason = cancellationReason;
1164  this.cancelledDataSourceModules = cancelledModules;
1165 
1166  this.processedFiles = processedFiles;
1167  this.estimatedFilesToProcess = estimatedFilesToProcess;
1168  this.snapShotTime = snapshotTime;
1169  this.tasksSnapshot = tasksSnapshot;
1170  }
1171 
1178  long getSnapshotTime() {
1179  return snapShotTime;
1180  }
1181 
1188  String getDataSource() {
1189  return dataSource;
1190  }
1191 
1198  long getJobId() {
1199  return this.jobId;
1200  }
1201 
1208  long getJobStartTime() {
1209  return jobStartTime;
1210  }
1211 
1212  DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() {
1213  return this.dataSourceLevelIngestModule;
1214  }
1215 
1216  boolean getFileIngestIsRunning() {
1217  return this.fileIngestRunning;
1218  }
1219 
1220  Date getFileIngestStartTime() {
1221  return this.fileIngestStartTime;
1222  }
1223 
1230  double getSpeed() {
1231  return (double) processedFiles / ((snapShotTime - jobStartTime) / 1000);
1232  }
1233 
1239  long getFilesProcessed() {
1240  return processedFiles;
1241  }
1242 
1249  long getFilesEstimated() {
1250  return estimatedFilesToProcess;
1251  }
1252 
1253  long getRootQueueSize() {
1254  if (null == this.tasksSnapshot) {
1255  return 0;
1256  }
1257  return this.tasksSnapshot.getRootQueueSize();
1258  }
1259 
1260  long getDirQueueSize() {
1261  if (null == this.tasksSnapshot) {
1262  return 0;
1263  }
1264  return this.tasksSnapshot.getDirectoryTasksQueueSize();
1265  }
1266 
1267  long getFileQueueSize() {
1268  if (null == this.tasksSnapshot) {
1269  return 0;
1270  }
1271  return this.tasksSnapshot.getFileQueueSize();
1272  }
1273 
1274  long getDsQueueSize() {
1275  if (null == this.tasksSnapshot) {
1276  return 0;
1277  }
1278  return this.tasksSnapshot.getDsQueueSize();
1279  }
1280 
1281  long getRunningListSize() {
1282  if (null == this.tasksSnapshot) {
1283  return 0;
1284  }
1285  return this.tasksSnapshot.getRunningListSize();
1286  }
1287 
1288  boolean isCancelled() {
1289  return this.jobCancelled;
1290  }
1291 
1297  IngestJob.CancellationReason getCancellationReason() {
1298  return this.jobCancellationReason;
1299  }
1300 
1308  List<String> getCancelledDataSourceIngestModules() {
1309  return Collections.unmodifiableList(this.cancelledDataSourceModules);
1310  }
1311 
1312  }
1313 
1314 }
static synchronized IngestManager getInstance()
void logIngestModuleErrors(List< IngestModuleError > errors)
List< IngestModuleTemplate > getEnabledIngestModuleTemplates()
static List< IngestModuleTemplate > getConfiguredIngestModuleTemplates(Map< String, IngestModuleTemplate > ingestModuleTemplates, List< String > pipelineConfig)
void addIngestModules(List< IngestModuleTemplate > templates, IngestModuleType type, SleuthkitCase skCase)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
final LinkedBlockingQueue< FileIngestPipeline > fileIngestPipelinesQueue
final List< FileIngestPipeline > fileIngestPipelines
volatile IngestJob.CancellationReason cancellationReason

Copyright © 2012-2018 Basis Technology. Generated on: Tue Dec 18 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.