Autopsy  4.9.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestManager.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2012-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import com.google.common.util.concurrent.ThreadFactoryBuilder;
22 import java.awt.EventQueue;
23 import java.beans.PropertyChangeEvent;
24 import java.beans.PropertyChangeListener;
25 import java.io.Serializable;
26 import java.util.ArrayList;
27 import java.util.Collection;
28 import java.util.Collections;
29 import java.util.Date;
30 import java.util.EnumSet;
31 import java.util.HashMap;
32 import java.util.HashSet;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.Set;
36 import java.util.concurrent.Callable;
37 import java.util.concurrent.ConcurrentHashMap;
38 import java.util.concurrent.ExecutorService;
39 import java.util.concurrent.Executors;
40 import java.util.concurrent.Future;
41 import java.util.concurrent.atomic.AtomicLong;
42 import java.util.logging.Level;
43 import java.util.stream.Collectors;
44 import java.util.stream.Stream;
45 import javax.annotation.concurrent.GuardedBy;
46 import javax.annotation.concurrent.Immutable;
47 import javax.annotation.concurrent.ThreadSafe;
48 import javax.swing.JOptionPane;
49 import org.netbeans.api.progress.ProgressHandle;
50 import org.openide.util.Cancellable;
51 import org.openide.util.NbBundle;
52 import org.openide.windows.WindowManager;
68 import org.sleuthkit.datamodel.AbstractFile;
69 import org.sleuthkit.datamodel.Content;
70 
110 @ThreadSafe
112 
113  private final static Logger logger = Logger.getLogger(IngestManager.class.getName());
114  private final static String INGEST_JOB_EVENT_CHANNEL_NAME = "%s-Ingest-Job-Events"; //NON-NLS
115  private final static Set<String> INGEST_JOB_EVENT_NAMES = Stream.of(IngestJobEvent.values()).map(IngestJobEvent::toString).collect(Collectors.toSet());
116  private final static String INGEST_MODULE_EVENT_CHANNEL_NAME = "%s-Ingest-Module-Events"; //NON-NLS
117  private final static Set<String> INGEST_MODULE_EVENT_NAMES = Stream.of(IngestModuleEvent.values()).map(IngestModuleEvent::toString).collect(Collectors.toSet());
118  private final static int MAX_ERROR_MESSAGE_POSTS = 200;
119  @GuardedBy("IngestManager.class")
120  private static IngestManager instance;
121  private final int numberOfFileIngestThreads;
122  private final AtomicLong nextIngestManagerTaskId = new AtomicLong(0L);
123  private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-start-ingest-jobs-%d").build()); //NON-NLS;
124  private final Map<Long, Future<Void>> startIngestJobFutures = new ConcurrentHashMap<>();
125  private final Map<Long, IngestJob> ingestJobsById = new HashMap<>();
126  private final ExecutorService dataSourceLevelIngestJobTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS;
127  private final ExecutorService fileLevelIngestJobTasksExecutor;
128  private final ExecutorService eventPublishingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-ingest-events-%d").build()); //NON-NLS;
132  private final AutopsyEventPublisher moduleEventPublisher = new AutopsyEventPublisher();
133  private final Object ingestMessageBoxLock = new Object();
134  private final AtomicLong ingestErrorMessagePosts = new AtomicLong(0L);
135  private final ConcurrentHashMap<Long, IngestThreadActivitySnapshot> ingestThreadActivitySnapshots = new ConcurrentHashMap<>();
136  private final ConcurrentHashMap<String, Long> ingestModuleRunTimes = new ConcurrentHashMap<>();
137  private volatile IngestMessageTopComponent ingestMessageBox;
138  private volatile boolean caseIsOpen;
139 
146  public synchronized static IngestManager getInstance() {
147  if (null == instance) {
148  instance = new IngestManager();
149  instance.subscribeToServiceMonitorEvents();
150  instance.subscribeToCaseEvents();
151  }
152  return instance;
153  }
154 
159  private IngestManager() {
160  /*
161  * Submit a single Runnable ingest manager task for processing data
162  * source level ingest job tasks to the data source level ingest job
163  * tasks executor.
164  */
165  long threadId = nextIngestManagerTaskId.incrementAndGet();
166  dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue()));
167  ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId));
168 
169  /*
170  * Submit a configurable number of Runnable ingest manager tasks for
171  * processing file level ingest job tasks to the file level ingest job
172  * tasks executor.
173  */
175  fileLevelIngestJobTasksExecutor = Executors.newFixedThreadPool(numberOfFileIngestThreads, new ThreadFactoryBuilder().setNameFormat("IM-file-ingest-%d").build()); //NON-NLS
176  for (int i = 0; i < numberOfFileIngestThreads; ++i) {
177  threadId = nextIngestManagerTaskId.incrementAndGet();
178  fileLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue()));
179  ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId));
180  }
181  }
182 
188  PropertyChangeListener propChangeListener = (PropertyChangeEvent evt) -> {
189  if (evt.getNewValue().equals(ServicesMonitor.ServiceStatus.DOWN.toString())) {
190  /*
191  * The application services considered to be key services are
192  * only necessary for multi-user cases.
193  */
194  try {
196  return;
197  }
198  } catch (NoCurrentCaseException noCaseOpenException) {
199  return;
200  }
201 
202  String serviceDisplayName = ServicesMonitor.Service.valueOf(evt.getPropertyName()).getDisplayName();
203  logger.log(Level.SEVERE, "Service {0} is down, cancelling all running ingest jobs", serviceDisplayName); //NON-NLS
205  EventQueue.invokeLater(new Runnable() {
206  @Override
207  public void run() {
208  JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(),
209  NbBundle.getMessage(this.getClass(), "IngestManager.cancellingIngest.msgDlg.text"),
210  NbBundle.getMessage(this.getClass(), "IngestManager.serviceIsDown.msgDlg.text", serviceDisplayName),
211  JOptionPane.ERROR_MESSAGE);
212  }
213  });
214  }
216  }
217  };
218 
219  /*
220  * The key services for multi-user cases are currently the case database
221  * server and the Solr server. The Solr server is a key service not
222  * because search is essential, but because the coordination service
223  * (ZooKeeper) is running embedded within the Solr server.
224  */
225  Set<String> servicesList = new HashSet<>();
226  servicesList.add(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString());
227  servicesList.add(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString());
228  this.servicesMonitor.addSubscriber(servicesList, propChangeListener);
229  }
230 
235  private void subscribeToCaseEvents() {
236  Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), (PropertyChangeEvent event) -> {
237  if (event.getNewValue() != null) {
238  handleCaseOpened();
239  } else {
240  handleCaseClosed();
241  }
242  });
243  }
244 
245  /*
246  * Handles a current case opened event by clearing the ingest messages inbox
247  * and opening a remote event channel for the current case.
248  *
249  * Note that current case change events are published in a strictly
250  * serialized manner, i.e., one event at a time, synchronously.
251  */
252  void handleCaseOpened() {
253  caseIsOpen = true;
255  try {
256  Case openedCase = Case.getCurrentCaseThrows();
257  String channelPrefix = openedCase.getName();
258  if (Case.CaseType.MULTI_USER_CASE == openedCase.getCaseType()) {
259  jobEventPublisher.openRemoteEventChannel(String.format(INGEST_JOB_EVENT_CHANNEL_NAME, channelPrefix));
260  moduleEventPublisher.openRemoteEventChannel(String.format(INGEST_MODULE_EVENT_CHANNEL_NAME, channelPrefix));
261  }
262  } catch (NoCurrentCaseException | AutopsyEventException ex) {
263  logger.log(Level.SEVERE, "Failed to open remote events channel", ex); //NON-NLS
264  MessageNotifyUtil.Notify.error(NbBundle.getMessage(IngestManager.class, "IngestManager.OpenEventChannel.Fail.Title"),
265  NbBundle.getMessage(IngestManager.class, "IngestManager.OpenEventChannel.Fail.ErrMsg"));
266  }
267  }
268 
269  /*
270  * Handles a current case closed event by cancelling all ingest jobs for the
271  * case, closing the remote event channel for the case, and clearing the
272  * ingest messages inbox.
273  *
274  * Note that current case change events are published in a strictly
275  * serialized manner, i.e., one event at a time, synchronously.
276  */
277  void handleCaseClosed() {
278  /*
279  * TODO (JIRA-2227): IngestManager should wait for cancelled ingest jobs
280  * to complete when a case is closed.
281  */
282  this.cancelAllIngestJobs(IngestJob.CancellationReason.CASE_CLOSED);
285  caseIsOpen = false;
287  }
288 
297  }
298 
305  public void queueIngestJob(Collection<Content> dataSources, IngestJobSettings settings) {
306  if (caseIsOpen) {
307  IngestJob job = new IngestJob(dataSources, settings, RuntimeProperties.runningWithGUI());
308  if (job.hasIngestPipeline()) {
309  long taskId = nextIngestManagerTaskId.incrementAndGet();
310  Future<Void> task = startIngestJobsExecutor.submit(new StartIngestJobTask(taskId, job));
311  startIngestJobFutures.put(taskId, task);
312  }
313  }
314  }
315 
324  public void queueIngestJob(Content dataSource, List<AbstractFile> files, IngestJobSettings settings) {
325  if (caseIsOpen) {
326  IngestJob job = new IngestJob(dataSource, files, settings, RuntimeProperties.runningWithGUI());
327  if (job.hasIngestPipeline()) {
328  long taskId = nextIngestManagerTaskId.incrementAndGet();
329  Future<Void> task = startIngestJobsExecutor.submit(new StartIngestJobTask(taskId, job));
330  startIngestJobFutures.put(taskId, task);
331  }
332  }
333  }
334 
344  public IngestJobStartResult beginIngestJob(Collection<Content> dataSources, IngestJobSettings settings) {
345  if (caseIsOpen) {
346  IngestJob job = new IngestJob(dataSources, settings, RuntimeProperties.runningWithGUI());
347  if (job.hasIngestPipeline()) {
348  return startIngestJob(job);
349  }
350  return new IngestJobStartResult(null, new IngestManagerException("No ingest pipeline created, likely due to no ingest modules being enabled"), null); //NON-NLS
351  }
352  return new IngestJobStartResult(null, new IngestManagerException("No case open"), null); //NON-NLS
353  }
354 
363  @NbBundle.Messages({
364  "IngestManager.startupErr.dlgTitle=Ingest Module Startup Failure",
365  "IngestManager.startupErr.dlgMsg=Unable to start up one or more ingest modules, ingest cancelled.",
366  "IngestManager.startupErr.dlgSolution=Please disable the failed modules or fix the errors before restarting ingest.",
367  "IngestManager.startupErr.dlgErrorList=Errors:"
368  })
370  List<IngestModuleError> errors = null;
371  Case openCase;
372  try {
373  openCase = Case.getCurrentCaseThrows();
374  } catch (NoCurrentCaseException ex) {
375  return new IngestJobStartResult(null, new IngestManagerException("Exception while getting open case.", ex), Collections.<IngestModuleError>emptyList()); //NON-NLS
376  }
377  if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) {
378  try {
381  EventQueue.invokeLater(new Runnable() {
382  @Override
383  public void run() {
384  String serviceDisplayName = ServicesMonitor.Service.REMOTE_CASE_DATABASE.getDisplayName();
385  JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(),
386  NbBundle.getMessage(this.getClass(), "IngestManager.cancellingIngest.msgDlg.text"),
387  NbBundle.getMessage(this.getClass(), "IngestManager.serviceIsDown.msgDlg.text", serviceDisplayName),
388  JOptionPane.ERROR_MESSAGE);
389  }
390  });
391  }
392  return new IngestJobStartResult(null, new IngestManagerException("Ingest aborted. Remote database is down"), Collections.<IngestModuleError>emptyList()); //NON-NLS
393  }
395  return new IngestJobStartResult(null, new IngestManagerException("Database server is down", ex), Collections.<IngestModuleError>emptyList()); //NON-NLS
396  }
397  }
398 
399  if (!ingestMonitor.isRunning()) {
400  ingestMonitor.start();
401  }
402 
403  synchronized (ingestJobsById) {
404  ingestJobsById.put(job.getId(), job);
405  }
406  errors = job.start();
407  if (errors.isEmpty()) {
408  this.fireIngestJobStarted(job.getId());
409  IngestManager.logger.log(Level.INFO, "Ingest job {0} started", job.getId()); //NON-NLS
410  } else {
411  synchronized (ingestJobsById) {
412  this.ingestJobsById.remove(job.getId());
413  }
414  for (IngestModuleError error : errors) {
415  logger.log(Level.SEVERE, String.format("Error starting %s ingest module for job %d", error.getModuleDisplayName(), job.getId()), error.getThrowable()); //NON-NLS
416  }
417  IngestManager.logger.log(Level.SEVERE, "Ingest job {0} could not be started", job.getId()); //NON-NLS
419  final StringBuilder message = new StringBuilder(1024);
420  message.append(Bundle.IngestManager_startupErr_dlgMsg()).append("\n"); //NON-NLS
421  message.append(Bundle.IngestManager_startupErr_dlgSolution()).append("\n\n"); //NON-NLS
422  message.append(Bundle.IngestManager_startupErr_dlgErrorList()).append("\n"); //NON-NLS
423  for (IngestModuleError error : errors) {
424  String moduleName = error.getModuleDisplayName();
425  String errorMessage = error.getThrowable().getLocalizedMessage();
426  message.append(moduleName).append(": ").append(errorMessage).append("\n"); //NON-NLS
427  }
428  message.append("\n\n");
429  EventQueue.invokeLater(() -> {
430  JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(), message, Bundle.IngestManager_startupErr_dlgTitle(), JOptionPane.ERROR_MESSAGE);
431  });
432  }
433  return new IngestJobStartResult(null, new IngestManagerException("Errors occurred while starting ingest"), errors); //NON-NLS
434  }
435 
436  return new IngestJobStartResult(job, null, errors);
437  }
438 
444  void finishIngestJob(IngestJob job) {
445  long jobId = job.getId();
446  synchronized (ingestJobsById) {
447  ingestJobsById.remove(jobId);
448  }
449  if (!job.isCancelled()) {
450  IngestManager.logger.log(Level.INFO, "Ingest job {0} completed", jobId); //NON-NLS
451  fireIngestJobCompleted(jobId);
452  } else {
453  IngestManager.logger.log(Level.INFO, "Ingest job {0} cancelled", jobId); //NON-NLS
454  fireIngestJobCancelled(jobId);
455  }
456  }
457 
464  public boolean isIngestRunning() {
465  synchronized (ingestJobsById) {
466  return !ingestJobsById.isEmpty();
467  }
468  }
469 
476  startIngestJobFutures.values().forEach((handle) -> {
477  handle.cancel(true);
478  });
479  synchronized (ingestJobsById) {
480  this.ingestJobsById.values().forEach((job) -> {
481  job.cancel(reason);
482  });
483  }
484  }
485 
491  public void addIngestJobEventListener(final PropertyChangeListener listener) {
492  jobEventPublisher.addSubscriber(INGEST_JOB_EVENT_NAMES, listener);
493  }
494 
500  public void removeIngestJobEventListener(final PropertyChangeListener listener) {
501  jobEventPublisher.removeSubscriber(INGEST_JOB_EVENT_NAMES, listener);
502  }
503 
509  public void addIngestModuleEventListener(final PropertyChangeListener listener) {
510  moduleEventPublisher.addSubscriber(INGEST_MODULE_EVENT_NAMES, listener);
511  }
512 
518  public void removeIngestModuleEventListener(final PropertyChangeListener listener) {
519  moduleEventPublisher.removeSubscriber(INGEST_MODULE_EVENT_NAMES, listener);
520  }
521 
527  void fireIngestJobStarted(long ingestJobId) {
528  AutopsyEvent event = new AutopsyEvent(IngestJobEvent.STARTED.toString(), ingestJobId, null);
529  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
530  }
531 
537  void fireIngestJobCompleted(long ingestJobId) {
538  AutopsyEvent event = new AutopsyEvent(IngestJobEvent.COMPLETED.toString(), ingestJobId, null);
539  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
540  }
541 
547  void fireIngestJobCancelled(long ingestJobId) {
548  AutopsyEvent event = new AutopsyEvent(IngestJobEvent.CANCELLED.toString(), ingestJobId, null);
549  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
550  }
551 
560  void fireDataSourceAnalysisStarted(long ingestJobId, long dataSourceIngestJobId, Content dataSource) {
561  AutopsyEvent event = new DataSourceAnalysisStartedEvent(ingestJobId, dataSourceIngestJobId, dataSource);
562  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
563  }
564 
573  void fireDataSourceAnalysisCompleted(long ingestJobId, long dataSourceIngestJobId, Content dataSource) {
574  AutopsyEvent event = new DataSourceAnalysisCompletedEvent(ingestJobId, dataSourceIngestJobId, dataSource, DataSourceAnalysisCompletedEvent.Reason.ANALYSIS_COMPLETED);
575  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
576  }
577 
586  void fireDataSourceAnalysisCancelled(long ingestJobId, long dataSourceIngestJobId, Content dataSource) {
587  AutopsyEvent event = new DataSourceAnalysisCompletedEvent(ingestJobId, dataSourceIngestJobId, dataSource, DataSourceAnalysisCompletedEvent.Reason.ANALYSIS_CANCELLED);
588  eventPublishingExecutor.submit(new PublishEventTask(event, jobEventPublisher));
589  }
590 
597  void fireFileIngestDone(AbstractFile file) {
598  AutopsyEvent event = new FileAnalyzedEvent(file);
599  eventPublishingExecutor.submit(new PublishEventTask(event, moduleEventPublisher));
600  }
601 
609  void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) {
610  AutopsyEvent event = new BlackboardPostEvent(moduleDataEvent);
611  eventPublishingExecutor.submit(new PublishEventTask(event, moduleEventPublisher));
612  }
613 
621  void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) {
622  AutopsyEvent event = new ContentChangedEvent(moduleContentEvent);
623  eventPublishingExecutor.submit(new PublishEventTask(event, moduleEventPublisher));
624  }
625 
631  void initIngestMessageInbox() {
632  synchronized (this.ingestMessageBoxLock) {
633  ingestMessageBox = IngestMessageTopComponent.findInstance();
634  }
635  }
636 
642  void postIngestMessage(IngestMessage message) {
643  synchronized (this.ingestMessageBoxLock) {
644  if (ingestMessageBox != null && RuntimeProperties.runningWithGUI()) {
645  if (message.getMessageType() != IngestMessage.MessageType.ERROR && message.getMessageType() != IngestMessage.MessageType.WARNING) {
646  ingestMessageBox.displayMessage(message);
647  } else {
648  long errorPosts = ingestErrorMessagePosts.incrementAndGet();
649  if (errorPosts <= MAX_ERROR_MESSAGE_POSTS) {
650  ingestMessageBox.displayMessage(message);
651  } else if (errorPosts == MAX_ERROR_MESSAGE_POSTS + 1) {
652  IngestMessage errorMessageLimitReachedMessage = IngestMessage.createErrorMessage(
653  NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.title"),
654  NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.subject"),
655  NbBundle.getMessage(this.getClass(), "IngestManager.IngestMessage.ErrorMessageLimitReached.msg", MAX_ERROR_MESSAGE_POSTS));
656  ingestMessageBox.displayMessage(errorMessageLimitReachedMessage);
657  }
658  }
659  }
660  }
661  }
662 
663  /*
664  * Clears the ingest messages inbox.
665  */
666  private void clearIngestMessageBox() {
667  synchronized (this.ingestMessageBoxLock) {
668  if (null != ingestMessageBox) {
669  ingestMessageBox.clearMessages();
670  }
672  }
673  }
674 
686  void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) {
687  ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource()));
688  }
689 
701  void setIngestTaskProgress(FileIngestTask task, String ingestModuleDisplayName) {
702  IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId());
703  IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile());
704  ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap);
705  incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime());
706  }
707 
715  void setIngestTaskProgressCompleted(DataSourceIngestTask task) {
716  ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId()));
717  }
718 
726  void setIngestTaskProgressCompleted(FileIngestTask task) {
727  IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId());
728  IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId());
729  ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap);
730  incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime());
731  }
732 
739  private void incrementModuleRunTime(String moduleDisplayName, Long duration) {
740  if (moduleDisplayName.equals("IDLE")) { //NON-NLS
741  return;
742  }
743 
744  synchronized (ingestModuleRunTimes) {
745  Long prevTimeL = ingestModuleRunTimes.get(moduleDisplayName);
746  long prevTime = 0;
747  if (prevTimeL != null) {
748  prevTime = prevTimeL;
749  }
750  prevTime += duration;
751  ingestModuleRunTimes.put(moduleDisplayName, prevTime);
752  }
753  }
754 
760  @Override
761  public Map<String, Long> getModuleRunTimes() {
762  synchronized (ingestModuleRunTimes) {
763  Map<String, Long> times = new HashMap<>(ingestModuleRunTimes);
764  return times;
765  }
766  }
767 
774  @Override
775  public List<IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots() {
776  return new ArrayList<>(ingestThreadActivitySnapshots.values());
777  }
778 
784  @Override
786  List<DataSourceIngestJob.Snapshot> snapShots = new ArrayList<>();
787  synchronized (ingestJobsById) {
788  ingestJobsById.values().forEach((job) -> {
789  snapShots.addAll(job.getDataSourceIngestJobSnapshots());
790  });
791  }
792  return snapShots;
793  }
794 
801  long getFreeDiskSpace() {
802  if (ingestMonitor != null) {
803  return ingestMonitor.getFreeSpace();
804  } else {
805  return -1;
806  }
807  }
808 
812  private final class StartIngestJobTask implements Callable<Void> {
813 
814  private final long threadId;
815  private final IngestJob job;
816  private ProgressHandle progress;
817 
818  StartIngestJobTask(long threadId, IngestJob job) {
819  this.threadId = threadId;
820  this.job = job;
821  }
822 
823  @Override
824  public Void call() {
825  try {
826  if (Thread.currentThread().isInterrupted()) {
827  synchronized (ingestJobsById) {
828  ingestJobsById.remove(job.getId());
829  }
830  return null;
831  }
832 
834  final String displayName = NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.displayName");
835  this.progress = ProgressHandle.createHandle(displayName, new Cancellable() {
836  @Override
837  public boolean cancel() {
838  if (progress != null) {
839  progress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.cancelling", displayName));
840  }
841  Future<?> handle = startIngestJobFutures.remove(threadId);
842  handle.cancel(true);
843  return true;
844  }
845  });
846  progress.start();
847  }
848 
849  startIngestJob(job);
850  return null;
851 
852  } finally {
853  if (null != progress) {
854  progress.finish();
855  }
856  startIngestJobFutures.remove(threadId);
857  }
858  }
859 
860  }
861 
865  private final class ExecuteIngestJobTasksTask implements Runnable {
866 
867  private final long threadId;
868  private final BlockingIngestTaskQueue tasks;
869 
870  ExecuteIngestJobTasksTask(long threadId, BlockingIngestTaskQueue tasks) {
871  this.threadId = threadId;
872  this.tasks = tasks;
873  }
874 
875  @Override
876  public void run() {
877  while (true) {
878  try {
879  IngestTask task = tasks.getNextTask(); // Blocks.
880  task.execute(threadId);
881  } catch (InterruptedException ex) {
882  break;
883  }
884  if (Thread.currentThread().isInterrupted()) {
885  break;
886  }
887  }
888  }
889  }
890 
894  private static final class PublishEventTask implements Runnable {
895 
896  private final AutopsyEvent event;
898 
907  this.event = event;
908  this.publisher = publisher;
909  }
910 
911  @Override
912  public void run() {
913  publisher.publish(event);
914  }
915 
916  }
917 
922  @Immutable
923  public static final class IngestThreadActivitySnapshot implements Serializable {
924 
925  private static final long serialVersionUID = 1L;
926 
927  private final long threadId;
928  private final Date startTime;
929  private final String activity;
930  private final String dataSourceName;
931  private final String fileName;
932  private final long jobId;
933 
941  IngestThreadActivitySnapshot(long threadId) {
942  this.threadId = threadId;
943  startTime = new Date();
944  this.activity = NbBundle.getMessage(this.getClass(), "IngestManager.IngestThreadActivitySnapshot.idleThread");
945  this.dataSourceName = "";
946  this.fileName = "";
947  this.jobId = 0;
948  }
949 
960  IngestThreadActivitySnapshot(long threadId, long jobId, String activity, Content dataSource) {
961  this.threadId = threadId;
962  this.jobId = jobId;
963  startTime = new Date();
964  this.activity = activity;
965  this.dataSourceName = dataSource.getName();
966  this.fileName = "";
967  }
968 
981  IngestThreadActivitySnapshot(long threadId, long jobId, String activity, Content dataSource, AbstractFile file) {
982  this.threadId = threadId;
983  this.jobId = jobId;
984  startTime = new Date();
985  this.activity = activity;
986  this.dataSourceName = dataSource.getName();
987  this.fileName = file.getName();
988  }
989 
995  long getIngestJobId() {
996  return jobId;
997  }
998 
1004  long getThreadId() {
1005  return threadId;
1006  }
1007 
1013  Date getStartTime() {
1014  return startTime;
1015  }
1016 
1022  String getActivity() {
1023  return activity;
1024  }
1025 
1033  String getDataSourceName() {
1034  return dataSourceName;
1035  }
1036 
1042  String getFileName() {
1043  return fileName;
1044  }
1045 
1046  }
1047 
1051  public enum IngestJobEvent {
1052 
1089  }
1090 
1094  public enum IngestModuleEvent {
1095 
1118  }
1119 
1123  public final static class IngestManagerException extends Exception {
1124 
1125  private static final long serialVersionUID = 1L;
1126 
1132  private IngestManagerException(String message) {
1133  super(message);
1134  }
1135 
1142  private IngestManagerException(String message, Throwable cause) {
1143  super(message, cause);
1144  }
1145  }
1146 
1155  @Deprecated
1156  public static void addPropertyChangeListener(final PropertyChangeListener listener) {
1159  }
1160 
1169  @Deprecated
1170  public static void removePropertyChangeListener(final PropertyChangeListener listener) {
1173  }
1174 
1185  @Deprecated
1186  public synchronized IngestJob startIngestJob(Collection<Content> dataSources, IngestJobSettings settings) {
1187  return beginIngestJob(dataSources, settings).getJob();
1188  }
1189 
1196  @Deprecated
1197  public void cancelAllIngestJobs() {
1199  }
1200 
1201 }
final ConcurrentHashMap< String, Long > ingestModuleRunTimes
final Map< Long, Future< Void > > startIngestJobFutures
void removeIngestModuleEventListener(final PropertyChangeListener listener)
List< IngestThreadActivitySnapshot > getIngestThreadActivitySnapshots()
void queueIngestJob(Content dataSource, List< AbstractFile > files, IngestJobSettings settings)
static synchronized IngestManager getInstance()
IngestJobStartResult startIngestJob(IngestJob job)
final ExecutorService dataSourceLevelIngestJobTasksExecutor
static void addPropertyChangeListener(final PropertyChangeListener listener)
IngestJobStartResult beginIngestJob(Collection< Content > dataSources, IngestJobSettings settings)
void addSubscriber(Set< String > eventNames, PropertyChangeListener subscriber)
static void removePropertyChangeListener(final PropertyChangeListener listener)
static final Set< String > INGEST_MODULE_EVENT_NAMES
volatile IngestMessageTopComponent ingestMessageBox
void addSubscriber(PropertyChangeListener subscriber)
void removeIngestJobEventListener(final PropertyChangeListener listener)
static final Set< String > INGEST_JOB_EVENT_NAMES
final AutopsyEventPublisher moduleEventPublisher
List< DataSourceIngestJob.Snapshot > getIngestJobSnapshots()
synchronized void openRemoteEventChannel(String channelName)
void addIngestJobEventListener(final PropertyChangeListener listener)
void queueIngestJob(Collection< Content > dataSources, IngestJobSettings settings)
void removeSubscriber(Set< String > eventNames, PropertyChangeListener subscriber)
void addIngestModuleEventListener(final PropertyChangeListener listener)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
void incrementModuleRunTime(String moduleDisplayName, Long duration)
static void addEventTypeSubscriber(Set< Events > eventTypes, PropertyChangeListener subscriber)
Definition: Case.java:429
synchronized IngestJob startIngestJob(Collection< Content > dataSources, IngestJobSettings settings)
final ConcurrentHashMap< Long, IngestThreadActivitySnapshot > ingestThreadActivitySnapshots
void cancelAllIngestJobs(IngestJob.CancellationReason reason)
final ExecutorService fileLevelIngestJobTasksExecutor
final Map< Long, IngestJob > ingestJobsById
final AutopsyEventPublisher jobEventPublisher

Copyright © 2012-2018 Basis Technology. Generated on: Tue Dec 18 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.