Autopsy  4.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestManager.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2013-2015 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import com.google.common.util.concurrent.ThreadFactoryBuilder;
22 import java.awt.EventQueue;
23 import java.beans.PropertyChangeEvent;
24 import java.beans.PropertyChangeListener;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.Date;
28 import java.util.HashMap;
29 import java.util.HashSet;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Set;
33 import java.util.concurrent.Callable;
34 import java.util.concurrent.ConcurrentHashMap;
35 import java.util.concurrent.ExecutorService;
36 import java.util.concurrent.Executors;
37 import java.util.concurrent.Future;
38 import java.util.concurrent.atomic.AtomicLong;
39 import java.util.logging.Level;
40 import java.util.stream.Collectors;
41 import java.util.stream.Stream;
42 import javax.swing.JOptionPane;
43 import org.netbeans.api.progress.ProgressHandle;
44 import org.netbeans.api.progress.ProgressHandleFactory;
45 import org.openide.util.Cancellable;
46 import org.openide.util.NbBundle;
61 import org.sleuthkit.datamodel.AbstractFile;
62 import org.sleuthkit.datamodel.Content;
63 
68 public class IngestManager {
69 
70  private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
71  private static IngestManager instance;
72  private final Object ingestMessageBoxLock = new Object();
73 
74  /*
75  * The ingest manager maintains a mapping of ingest job ids to running
76  * ingest jobs.
77  */
78  private final Map<Long, IngestJob> jobsById;
79 
80  /*
81  * Each runnable/callable task the ingest manager submits to its thread
82  * pools is given a unique thread/task ID.
83  */
84  private final AtomicLong nextThreadId;
85 
86  /*
87  * Ingest jobs may be queued to be started on a pool thread by start ingest
88  * job tasks. A mapping of task ids to the Future objects for each task is
89  * maintained to allow for task cancellation.
90  */
91  private final Map<Long, Future<Void>> startIngestJobTasks;
92  private final ExecutorService startIngestJobsThreadPool;
93 
94  /*
95  * Ingest jobs use an ingest task scheduler to break themselves down into
96  * data source level and file level tasks. The ingest scheduler puts these
97  * ingest tasks into queues for execution on ingest manager pool threads by
98  * ingest task executers. There is a single data source level ingest thread
99  * and a user configurable number of file level ingest threads.
100  */
101  private final ExecutorService dataSourceIngestThreadPool;
102  private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1;
103  private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16;
104  private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2;
106  private final ExecutorService fileIngestThreadPool;
107 
108  private static final String JOB_EVENT_CHANNEL_NAME = "%s-Ingest-Job-Events"; //NON-NLS
109  private static final String MODULE_EVENT_CHANNEL_NAME = "%s-Ingest-Module-Events"; //NON-NLS
110  private static final Set<String> jobEventNames = Stream.of(IngestJobEvent.values())
111  .map(IngestJobEvent::toString)
112  .collect(Collectors.toSet());
113  private static final Set<String> moduleEventNames = Stream.of(IngestModuleEvent.values())
114  .map(IngestModuleEvent::toString)
115  .collect(Collectors.toSet());
118  private final ExecutorService eventPublishingExecutor;
119 
120  /*
121  * The ingest manager uses an ingest monitor to determine when system
122  * resources are under pressure. If the monitor detects such a situation, it
123  * calls back to the ingest manager to cancel all ingest jobs in progress.
124  */
126 
127  /*
128  * The ingest manager provides access to a top component that is used by
129  * ingest module to post messages for the user. A count of the posts is used
130  * as a cap to avoid bogging down the application.
131  */
132  private static final int MAX_ERROR_MESSAGE_POSTS = 200;
133  private volatile IngestMessageTopComponent ingestMessageBox;
134  private final AtomicLong ingestErrorMessagePosts;
135 
136  /*
137  * The ingest manager supports reporting of ingest processing progress by
138  * collecting snapshots of the activities of the ingest threads, ingest job
139  * progress, and ingest module run times.
140  */
141  private final ConcurrentHashMap<Long, IngestThreadActivitySnapshot> ingestThreadActivitySnapshots;
142  private final ConcurrentHashMap<String, Long> ingestModuleRunTimes;
143 
144  /*
145  * The ingest job creation capability of the ingest manager can be turned on
146  * and off to support an orderly shut down of the application.
147  */
148  private volatile boolean jobCreationIsEnabled;
149 
150  /*
151  * Ingest manager subscribes to service outage notifications. If key
152  * services are down, ingest manager cancels all ingest jobs in progress.
153  */
155 
159  public enum IngestJobEvent {
160 
197  };
198 
202  public enum IngestModuleEvent {
203 
226  };
227 
234  public synchronized static IngestManager getInstance() {
235  if (instance == null) {
241  instance = new IngestManager();
242  instance.subscribeToCaseEvents();
243  }
244  return instance;
245  }
246 
255  private IngestManager() {
256  this.ingestModuleRunTimes = new ConcurrentHashMap<>();
257  this.ingestThreadActivitySnapshots = new ConcurrentHashMap<>();
258  this.ingestErrorMessagePosts = new AtomicLong(0L);
259  this.ingestMonitor = new IngestMonitor();
260  this.eventPublishingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-ingest-events-%d").build()); //NON-NLS
261  this.jobEventPublisher = new AutopsyEventPublisher();
262  this.moduleEventPublisher = new AutopsyEventPublisher();
263  this.dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS
264  this.startIngestJobsThreadPool = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-start-ingest-jobs-%d").build()); //NON-NLS
265  this.nextThreadId = new AtomicLong(0L);
266  this.jobsById = new HashMap<>();
267  this.startIngestJobTasks = new ConcurrentHashMap<>();
268 
269  this.servicesMonitor = ServicesMonitor.getInstance();
271 
273 
274  numberOfFileIngestThreads = UserPreferences.numberOfFileIngestThreads();
275  if ((numberOfFileIngestThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfFileIngestThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) {
276  numberOfFileIngestThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS;
277  UserPreferences.setNumberOfFileIngestThreads(numberOfFileIngestThreads);
278  }
279  fileIngestThreadPool = Executors.newFixedThreadPool(numberOfFileIngestThreads, new ThreadFactoryBuilder().setNameFormat("IM-file-ingest-%d").build()); //NON-NLS
280  for (int i = 0; i < numberOfFileIngestThreads; ++i) {
282  }
283  }
284 
290  long threadId = nextThreadId.incrementAndGet();
291  dataSourceIngestThreadPool.submit(new ExecuteIngestJobsTask(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue()));
292  ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId));
293  }
294 
299  private void startFileIngestThread() {
300  long threadId = nextThreadId.incrementAndGet();
301  fileIngestThreadPool.submit(new ExecuteIngestJobsTask(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue()));
302  ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId));
303  }
304 
308  private void subscribeToCaseEvents() {
309  Case.addEventSubscriber(Case.Events.CURRENT_CASE.toString(), new PropertyChangeListener() {
310  @Override
311  public void propertyChange(PropertyChangeEvent event) {
312  if (event.getNewValue() != null) {
313  handleCaseOpened();
314  } else {
315  handleCaseClosed();
316  }
317  }
318  });
319  }
320 
326  PropertyChangeListener propChangeListener = new PropertyChangeListener() {
327  @Override
328  public void propertyChange(PropertyChangeEvent evt) {
329  if (evt.getNewValue().equals(ServicesMonitor.ServiceStatus.DOWN.toString())) {
330 
331  // check whether a multi-user case is currently being processed
332  try {
334  return;
335  }
336  } catch (IllegalStateException ignore) {
337  // thorown by Case.getCurrentCase() when no case is open
338  return;
339  }
340 
341  // one of the services we subscribed to went down
342  String serviceDisplayName = ServicesMonitor.Service.valueOf(evt.getPropertyName()).getDisplayName();
343  logger.log(Level.SEVERE, "Service {0} is down! Cancelling all running ingest jobs", serviceDisplayName); //NON-NLS
344 
345  // display notification if running interactively
347  EventQueue.invokeLater(new Runnable() {
348  @Override
349  public void run() {
350  JOptionPane.showMessageDialog(null,
351  NbBundle.getMessage(this.getClass(), "IngestManager.cancellingIngest.msgDlg.text"),
352  NbBundle.getMessage(this.getClass(), "IngestManager.serviceIsDown.msgDlg.text", serviceDisplayName),
353  JOptionPane.ERROR_MESSAGE);
354  }
355  });
356  }
357 
358  // cancel ingest if running
360  }
361  }
362  };
363 
364  // subscribe to services of interest
365  Set<String> servicesList = new HashSet<>();
366  servicesList.add(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString());
367  servicesList.add(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString());
368  this.servicesMonitor.addSubscriber(servicesList, propChangeListener);
369  }
370 
371  synchronized void handleCaseOpened() {
372  this.jobCreationIsEnabled = true;
374  try {
381  Case openedCase = Case.getCurrentCase();
382  String channelPrefix = openedCase.getTextIndexName();
383  if (Case.CaseType.MULTI_USER_CASE == openedCase.getCaseType()) {
384  jobEventPublisher.openRemoteEventChannel(String.format(JOB_EVENT_CHANNEL_NAME, channelPrefix));
385  moduleEventPublisher.openRemoteEventChannel(String.format(MODULE_EVENT_CHANNEL_NAME, channelPrefix));
386  }
387  } catch (IllegalStateException | AutopsyEventException ex) {
388  logger.log(Level.SEVERE, "Failed to open remote events channel", ex); //NON-NLS
389  MessageNotifyUtil.Notify.error(NbBundle.getMessage(IngestManager.class, "IngestManager.OpenEventChannel.Fail.Title"),
390  NbBundle.getMessage(IngestManager.class, "IngestManager.OpenEventChannel.Fail.ErrMsg"));
391  }
392  }
393 
394  synchronized void handleCaseClosed() {
395  jobEventPublisher.closeRemoteEventChannel();
396  moduleEventPublisher.closeRemoteEventChannel();
397  this.jobCreationIsEnabled = false;
399  }
400 
408  @Deprecated
409  public synchronized void setRunInteractively(boolean runInteractively) {
410  RuntimeProperties.setCoreComponentsActive(runInteractively);
411  }
412 
418  void initIngestMessageInbox() {
419  synchronized (this.ingestMessageBoxLock) {
420  ingestMessageBox = IngestMessageTopComponent.findInstance();
421  }
422  }
423 
429  void postIngestMessage(IngestMessage message) {
430  synchronized (this.ingestMessageBoxLock) {
431  if (ingestMessageBox != null && RuntimeProperties.coreComponentsAreActive()) {
432  if (message.getMessageType() != IngestMessage.MessageType.ERROR && message.getMessageType() != IngestMessage.MessageType.WARNING) {
433  ingestMessageBox.displayMessage(message);
434  } else {
435  long errorPosts = ingestErrorMessagePosts.incrementAndGet();
436  if (errorPosts <= MAX_ERROR_MESSAGE_POSTS) {
437  ingestMessageBox.displayMessage(message);
438  } else if (errorPosts == MAX_ERROR_MESSAGE_POSTS + 1) {
439  IngestMessage errorMessageLimitReachedMessage = IngestMessage.createErrorMessage(
440  NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.title"),
441  NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.subject"),
442  NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.msg", MAX_ERROR_MESSAGE_POSTS));
443  ingestMessageBox.displayMessage(errorMessageLimitReachedMessage);
444  }
445  }
446  }
447  }
448  }
449 
450  private void clearIngestMessageBox() {
451  synchronized (this.ingestMessageBoxLock) {
452  if (ingestMessageBox != null) {
453  ingestMessageBox.clearMessages();
454  }
455  ingestErrorMessagePosts.set(0);
456  }
457  }
458 
467  }
468 
476  public void queueIngestJob(Collection<Content> dataSources, IngestJobSettings settings) {
477  if (jobCreationIsEnabled) {
478  IngestJob job = new IngestJob(dataSources, settings, RuntimeProperties.coreComponentsAreActive());
479  if (job.hasIngestPipeline()) {
480  long taskId = nextThreadId.incrementAndGet();
481  Future<Void> task = startIngestJobsThreadPool.submit(new StartIngestJobTask(taskId, job));
482  startIngestJobTasks.put(taskId, task);
483  }
484  }
485  }
486 
495  public synchronized IngestJob startIngestJob(Collection<Content> dataSources, IngestJobSettings settings) {
496  if (this.jobCreationIsEnabled) {
497  IngestJob job = new IngestJob(dataSources, settings, RuntimeProperties.coreComponentsAreActive());
498  if (job.hasIngestPipeline()) {
499  if (this.startIngestJob(job)) {
500  return job;
501  }
502  }
503  }
504  return null;
505  }
506 
514  private boolean startIngestJob(IngestJob job) {
515  boolean success = false;
516  if (this.jobCreationIsEnabled) {
517  // multi-user cases must have multi-user database service running
519  try {
520  if (!servicesMonitor.getServiceStatus(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString()).equals(ServicesMonitor.ServiceStatus.UP.toString())) {
521  // display notification if running interactively
523  EventQueue.invokeLater(new Runnable() {
524  @Override
525  public void run() {
526  String serviceDisplayName = ServicesMonitor.Service.REMOTE_CASE_DATABASE.getDisplayName();
527  JOptionPane.showMessageDialog(null,
528  NbBundle.getMessage(this.getClass(), "IngestManager.cancellingIngest.msgDlg.text"),
529  NbBundle.getMessage(this.getClass(), "IngestManager.serviceIsDown.msgDlg.text", serviceDisplayName),
530  JOptionPane.ERROR_MESSAGE);
531  }
532  });
533  }
534  // abort ingest
535  return false;
536  }
537  } catch (ServicesMonitor.ServicesMonitorException ignore) {
538  return false;
539  }
540  }
541 
542  if (!ingestMonitor.isRunning()) {
543  ingestMonitor.start();
544  }
545 
546  synchronized (jobsById) {
547  jobsById.put(job.getId(), job);
548  }
549  List<IngestModuleError> errors = job.start();
550  if (errors.isEmpty()) {
551  this.fireIngestJobStarted(job.getId());
552  IngestManager.logger.log(Level.INFO, "Ingest job {0} started", job.getId()); //NON-NLS
553  success = true;
554  } else {
555  synchronized (jobsById) {
556  this.jobsById.remove(job.getId());
557  }
558  for (IngestModuleError error : errors) {
559  logger.log(Level.SEVERE, String.format("Error starting %s ingest module", error.getModuleDisplayName()), error.getModuleError()); //NON-NLS
560  }
561  IngestManager.logger.log(Level.INFO, "Ingest job {0} could not be started", job.getId()); //NON-NLS
563  EventQueue.invokeLater(new Runnable() {
564 
565  @Override
566  public void run() {
567  StringBuilder moduleStartUpErrors = new StringBuilder();
568  for (IngestModuleError error : errors) {
569  String moduleName = error.getModuleDisplayName();
570  moduleStartUpErrors.append(moduleName);
571  moduleStartUpErrors.append(": ");
572  moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage());
573  moduleStartUpErrors.append("\n");
574  }
575  StringBuilder notifyMessage = new StringBuilder();
576  notifyMessage.append(NbBundle.getMessage(this.getClass(),
577  "IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg"));
578  notifyMessage.append("\n");
579  notifyMessage.append(NbBundle.getMessage(this.getClass(),
580  "IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution"));
581  notifyMessage.append("\n");
582  notifyMessage.append(NbBundle.getMessage(this.getClass(),
583  "IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList",
584  moduleStartUpErrors.toString()));
585  notifyMessage.append("\n\n");
586  JOptionPane.showMessageDialog(null, notifyMessage.toString(),
587  NbBundle.getMessage(this.getClass(),
588  "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE);
589  }
590  });
591  }
592  }
593  }
594  return success;
595  }
596 
597  synchronized void finishIngestJob(IngestJob job) {
598  long jobId = job.getId();
599  synchronized (jobsById) {
600  jobsById.remove(jobId);
601  }
602  if (!job.isCancelled()) {
603  IngestManager.logger.log(Level.INFO, "Ingest job {0} completed", jobId); //NON-NLS
604  fireIngestJobCompleted(jobId);
605  } else {
606  IngestManager.logger.log(Level.INFO, "Ingest job {0} cancelled", jobId); //NON-NLS
607  fireIngestJobCancelled(jobId);
608  }
609  }
610 
616  public boolean isIngestRunning() {
617  synchronized (jobsById) {
618  return !jobsById.isEmpty();
619  }
620  }
621 
628  @Deprecated
629  public void cancelAllIngestJobs() {
631  }
632 
639  /*
640  * Cancel the start job tasks.
641  */
642  for (Future<Void> handle : startIngestJobTasks.values()) {
643  handle.cancel(true);
644  }
645 
646  /*
647  * Cancel the jobs in progress.
648  */
649  synchronized (jobsById) {
650  for (IngestJob job : this.jobsById.values()) {
651  job.cancel(reason);
652  }
653  }
654  }
655 
661  public void addIngestJobEventListener(final PropertyChangeListener listener) {
662  jobEventPublisher.addSubscriber(jobEventNames, listener);
663  }
664 
670  public void removeIngestJobEventListener(final PropertyChangeListener listener) {
671  jobEventPublisher.removeSubscriber(jobEventNames, listener);
672  }
673 
679  public void addIngestModuleEventListener(final PropertyChangeListener listener) {
680  moduleEventPublisher.addSubscriber(moduleEventNames, listener);
681  }
682 
688  public void removeIngestModuleEventListener(final PropertyChangeListener listener) {
689  moduleEventPublisher.removeSubscriber(moduleEventNames, listener);
690  }
691 
700  @Deprecated
701  public static void addPropertyChangeListener(final PropertyChangeListener listener) {
702  instance.jobEventPublisher.addSubscriber(jobEventNames, listener);
703  instance.moduleEventPublisher.addSubscriber(moduleEventNames, listener);
704  }
705 
714  @Deprecated
715  public static void removePropertyChangeListener(final PropertyChangeListener listener) {
716  instance.jobEventPublisher.removeSubscriber(jobEventNames, listener);
717  instance.moduleEventPublisher.removeSubscriber(moduleEventNames, listener);
718  }
719 
725  void fireIngestJobStarted(long ingestJobId) {
726  AutopsyEvent event = new AutopsyEvent(IngestJobEvent.STARTED.toString(), ingestJobId, null);
727  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
728  }
729 
735  void fireIngestJobCompleted(long ingestJobId) {
736  AutopsyEvent event = new AutopsyEvent(IngestJobEvent.COMPLETED.toString(), ingestJobId, null);
737  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
738  }
739 
745  void fireIngestJobCancelled(long ingestJobId) {
746  AutopsyEvent event = new AutopsyEvent(IngestJobEvent.CANCELLED.toString(), ingestJobId, null);
747  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
748  }
749 
757  void fireDataSourceAnalysisStarted(long ingestJobId, long dataSourceIngestJobId, Content dataSource) {
758  AutopsyEvent event = new DataSourceAnalysisStartedEvent(ingestJobId, dataSourceIngestJobId, dataSource);
759  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
760  }
761 
769  void fireDataSourceAnalysisCompleted(long ingestJobId, long dataSourceIngestJobId, Content dataSource) {
770  AutopsyEvent event = new DataSourceAnalysisCompletedEvent(ingestJobId, dataSourceIngestJobId, dataSource, DataSourceAnalysisCompletedEvent.Reason.ANALYSIS_COMPLETED);
771  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
772  }
773 
781  void fireDataSourceAnalysisCancelled(long ingestJobId, long dataSourceIngestJobId, Content dataSource) {
782  AutopsyEvent event = new DataSourceAnalysisCompletedEvent(ingestJobId, dataSourceIngestJobId, dataSource, DataSourceAnalysisCompletedEvent.Reason.ANALYSIS_CANCELLED);
783  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
784  }
785 
791  void fireFileIngestDone(AbstractFile file) {
792  AutopsyEvent event = new FileAnalyzedEvent(file);
793  eventPublishingExecutor.submit(new PublishEventTask(event, moduleEventPublisher));
794  }
795 
801  void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) {
802  AutopsyEvent event = new BlackboardPostEvent(moduleDataEvent);
803  eventPublishingExecutor.submit(new PublishEventTask(event, moduleEventPublisher));
804  }
805 
813  void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) {
814  AutopsyEvent event = new ContentChangedEvent(moduleContentEvent);
815  eventPublishingExecutor.submit(new PublishEventTask(event, moduleEventPublisher));
816  }
817 
824  void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) {
825  ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource()));
826  }
827 
834  void setIngestTaskProgress(FileIngestTask task, String ingestModuleDisplayName) {
835  IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId());
836  IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile());
837  ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap);
838 
839  incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime());
840  }
841 
847  void setIngestTaskProgressCompleted(DataSourceIngestTask task) {
848  ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId()));
849  }
850 
856  void setIngestTaskProgressCompleted(FileIngestTask task) {
857  IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId());
858  IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId());
859  ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap);
860  incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime());
861  }
862 
869  private void incrementModuleRunTime(String moduleName, Long duration) {
870  if (moduleName.equals("IDLE")) { //NON-NLS
871  return;
872  }
873 
874  synchronized (ingestModuleRunTimes) {
875  Long prevTimeL = ingestModuleRunTimes.get(moduleName);
876  long prevTime = 0;
877  if (prevTimeL != null) {
878  prevTime = prevTimeL;
879  }
880  prevTime += duration;
881  ingestModuleRunTimes.put(moduleName, prevTime);
882  }
883  }
884 
890  Map<String, Long> getModuleRunTimes() {
891  synchronized (ingestModuleRunTimes) {
892  Map<String, Long> times = new HashMap<>(ingestModuleRunTimes);
893  return times;
894  }
895  }
896 
902  List<IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots() {
903  return new ArrayList<>(ingestThreadActivitySnapshots.values());
904  }
905 
911  List<DataSourceIngestJob.Snapshot> getIngestJobSnapshots() {
912  List<DataSourceIngestJob.Snapshot> snapShots = new ArrayList<>();
913  synchronized (jobsById) {
914  for (IngestJob job : jobsById.values()) {
915  snapShots.addAll(job.getDataSourceIngestJobSnapshots());
916  }
917  }
918  return snapShots;
919  }
920 
927  long getFreeDiskSpace() {
928  if (ingestMonitor != null) {
929  return ingestMonitor.getFreeSpace();
930  } else {
931  return -1;
932  }
933  }
934 
938  private final class StartIngestJobTask implements Callable<Void> {
939 
940  private final long threadId;
941  private final IngestJob job;
942  private ProgressHandle progress;
943 
944  StartIngestJobTask(long threadId, IngestJob job) {
945  this.threadId = threadId;
946  this.job = job;
947  }
948 
949  @Override
950  public Void call() {
951  try {
952  if (Thread.currentThread().isInterrupted()) {
953  synchronized (jobsById) {
954  jobsById.remove(job.getId());
955  }
956  return null;
957  }
958 
960  final String displayName = NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.displayName");
961  this.progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
962  @Override
963  public boolean cancel() {
964  if (progress != null) {
965  progress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.cancelling", displayName));
966  }
967  Future<?> handle = startIngestJobTasks.remove(threadId);
968  handle.cancel(true);
969  return true;
970  }
971  });
972  progress.start();
973  }
974 
975  startIngestJob(job);
976  return null;
977 
978  } finally {
979  if (null != progress) {
980  progress.finish();
981  }
982  startIngestJobTasks.remove(threadId);
983  }
984  }
985 
986  }
987 
991  private final class ExecuteIngestJobsTask implements Runnable {
992 
993  private final long threadId;
994  private final IngestTaskQueue tasks;
995 
996  ExecuteIngestJobsTask(long threadId, IngestTaskQueue tasks) {
997  this.threadId = threadId;
998  this.tasks = tasks;
999  }
1000 
1001  @Override
1002  public void run() {
1003  while (true) {
1004  try {
1005  IngestTask task = tasks.getNextTask(); // Blocks.
1006  task.execute(threadId);
1007  } catch (InterruptedException ex) {
1008  break;
1009  }
1010  if (Thread.currentThread().isInterrupted()) {
1011  break;
1012  }
1013  }
1014  }
1015  }
1016 
1020  private static final class PublishEventTask implements Runnable {
1021 
1022  private final AutopsyEvent event;
1024 
1033  this.event = event;
1034  this.publisher = publisher;
1035  }
1036 
1040  @Override
1041  public void run() {
1042  publisher.publish(event);
1043  }
1044 
1045  }
1046 
1047  static final class IngestThreadActivitySnapshot {
1048 
1049  private final long threadId;
1050  private final Date startTime;
1051  private final String activity;
1052  private final String dataSourceName;
1053  private final String fileName;
1054  private final long jobId;
1055 
1056  // nothing is running on the thread
1057  IngestThreadActivitySnapshot(long threadId) {
1058  this.threadId = threadId;
1059  startTime = new Date();
1060  this.activity = NbBundle.getMessage(this.getClass(), "IngestManager.IngestThreadActivitySnapshot.idleThread");
1061  this.dataSourceName = "";
1062  this.fileName = "";
1063  this.jobId = 0;
1064  }
1065 
1066  // data souce thread
1067  IngestThreadActivitySnapshot(long threadId, long jobId, String activity, Content dataSource) {
1068  this.threadId = threadId;
1069  this.jobId = jobId;
1070  startTime = new Date();
1071  this.activity = activity;
1072  this.dataSourceName = dataSource.getName();
1073  this.fileName = "";
1074  }
1075 
1076  // file ingest thread
1077  IngestThreadActivitySnapshot(long threadId, long jobId, String activity, Content dataSource, AbstractFile file) {
1078  this.threadId = threadId;
1079  this.jobId = jobId;
1080  startTime = new Date();
1081  this.activity = activity;
1082  this.dataSourceName = dataSource.getName();
1083  this.fileName = file.getName();
1084  }
1085 
1086  long getJobId() {
1087  return jobId;
1088  }
1089 
1090  long getThreadId() {
1091  return threadId;
1092  }
1093 
1094  Date getStartTime() {
1095  return startTime;
1096  }
1097 
1098  String getActivity() {
1099  return activity;
1100  }
1101 
1102  String getDataSourceName() {
1103  return dataSourceName;
1104  }
1105 
1106  String getFileName() {
1107  return fileName;
1108  }
1109 
1110  }
1111 
1112 }
final ConcurrentHashMap< String, Long > ingestModuleRunTimes
void removeIngestModuleEventListener(final PropertyChangeListener listener)
static synchronized IngestManager getInstance()
final Map< Long, Future< Void > > startIngestJobTasks
static void addPropertyChangeListener(final PropertyChangeListener listener)
void addSubscriber(Set< String > eventNames, PropertyChangeListener subscriber)
static void setCoreComponentsActive(boolean coreComponentsActive)
static void removePropertyChangeListener(final PropertyChangeListener listener)
volatile IngestMessageTopComponent ingestMessageBox
void addSubscriber(PropertyChangeListener subscriber)
static synchronized ServicesMonitor getInstance()
void removeIngestJobEventListener(final PropertyChangeListener listener)
void incrementModuleRunTime(String moduleName, Long duration)
void addIngestJobEventListener(final PropertyChangeListener listener)
final Map< Long, IngestJob > jobsById
synchronized boolean isCancelled()
Definition: IngestJob.java:243
void queueIngestJob(Collection< Content > dataSources, IngestJobSettings settings)
void removeSubscriber(Set< String > eventNames, PropertyChangeListener subscriber)
static final Set< String > jobEventNames
static final Set< String > moduleEventNames
void addIngestModuleEventListener(final PropertyChangeListener listener)
synchronized static Logger getLogger(String name)
Definition: Logger.java:166
synchronized IngestJob startIngestJob(Collection< Content > dataSources, IngestJobSettings settings)
final ConcurrentHashMap< Long, IngestThreadActivitySnapshot > ingestThreadActivitySnapshots
void cancelAllIngestJobs(IngestJob.CancellationReason reason)
synchronized void setRunInteractively(boolean runInteractively)
static void addEventSubscriber(Set< String > eventNames, PropertyChangeListener subscriber)
Definition: Case.java:1318

Copyright © 2012-2015 Basis Technology. Generated on: Wed Apr 6 2016
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.