Autopsy  4.19.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestJob.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.Date;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.concurrent.ConcurrentHashMap;
29 import java.util.concurrent.atomic.AtomicInteger;
30 import java.util.concurrent.atomic.AtomicLong;
31 import java.util.logging.Level;
32 import org.openide.util.NbBundle;
34 import org.sleuthkit.datamodel.AbstractFile;
35 import org.sleuthkit.datamodel.Content;
36 
41 public final class IngestJob {
42 
43  /*
44  * An ingest job can be cancelled for various reasons.
45  */
46  public enum CancellationReason {
47 
48  NOT_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.notCancelled.text")),
49  USER_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.cancelledByUser.text")),
50  INGEST_MODULES_STARTUP_FAILED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.ingestModStartFail.text")),
51  OUT_OF_DISK_SPACE(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.outOfDiskSpace.text")),
52  SERVICES_DOWN(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.servicesDown.text")),
53  CASE_CLOSED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.caseClosed.text"));
54 
55  private final String displayName;
56 
57  private CancellationReason(String displayName) {
58  this.displayName = displayName;
59  }
60 
61  public String getDisplayName() {
62  return displayName;
63  }
64  }
65 
69  enum Mode {
70  BATCH,
71  STREAMING
72  }
73 
74  private static final Logger logger = Logger.getLogger(IngestJob.class.getName());
75  private final static AtomicLong nextId = new AtomicLong(0L);
76  private final long id;
77  private final List<Content> dataSources = new ArrayList<>();
78  private final List<AbstractFile> files = new ArrayList<>();
79  private final Mode ingestMode;
80  private final Map<Long, IngestJobPipeline> ingestJobPipelines;
81  private final AtomicInteger incompleteJobsCount;
82  private final IngestJobSettings settings;
84 
92  IngestJob(Collection<Content> dataSources, IngestJobSettings settings) {
93  this.id = IngestJob.nextId.getAndIncrement();
94  this.settings = settings;
95  this.ingestJobPipelines = new ConcurrentHashMap<>();
96  this.ingestMode = Mode.BATCH;
97  this.dataSources.addAll(dataSources);
98  incompleteJobsCount = new AtomicInteger(dataSources.size());
99  cancellationReason = CancellationReason.NOT_CANCELLED;
100  }
101 
111  IngestJob(Content dataSource, List<AbstractFile> files, IngestJobSettings settings) {
112  this(Arrays.asList(dataSource), settings);
113  this.files.addAll(files);
114  }
115 
122  IngestJob(Content dataSource, Mode ingestMode, IngestJobSettings settings) {
123  this.id = IngestJob.nextId.getAndIncrement();
124  this.ingestJobPipelines = new ConcurrentHashMap<>();
125  this.dataSources.add(dataSource);
126  this.settings = settings;
127  this.ingestMode = ingestMode;
128  incompleteJobsCount = new AtomicInteger(1);
129  cancellationReason = CancellationReason.NOT_CANCELLED;
130  }
131 
137  public long getId() {
138  return this.id;
139  }
140 
148  boolean hasIngestPipeline() {
149  return (!settings.getEnabledIngestModuleTemplates().isEmpty());
150  }
151 
157  void addStreamingIngestFiles(List<Long> fileObjIds) {
158  if (ingestJobPipelines.isEmpty()) {
159  logger.log(Level.SEVERE, "Attempted to add streaming ingest files with no IngestJobPipeline");
160  return;
161  }
162  // Streaming ingest jobs will only have one data source
163  IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next();
164  streamingIngestPipeline.addStreamedFiles(fileObjIds);
165  }
166 
170  void processStreamingIngestDataSource() {
171  if (ingestJobPipelines.isEmpty()) {
172  logger.log(Level.SEVERE, "Attempted to start data source ingest with no IngestJobPipeline");
173  return;
174  }
175  // Streaming ingest jobs will only have one data source
176  IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next();
177  streamingIngestPipeline.addStreamedDataSource();
178  }
179 
186  List<IngestModuleError> start() throws InterruptedException {
187  /*
188  * Set up the ingest job pipelines, one for each data source to be
189  * ingested by this job.
190  */
191  if (files.isEmpty()) {
192  for (Content dataSource : dataSources) {
193  IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSource, settings);
194  ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline);
195  }
196  } else {
197  IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSources.get(0), files, settings);
198  ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline);
199  }
200  incompleteJobsCount.set(ingestJobPipelines.size());
201 
202  /*
203  * Try to start up each ingest job pipeline. Stop at the first failure.
204  */
205  List<IngestModuleError> errors = new ArrayList<>();
206  for (IngestJobPipeline ingestJobPipeline : ingestJobPipelines.values()) {
207  errors.addAll(ingestJobPipeline.startUp());
208  if (errors.isEmpty() == false) {
209  break;
210  }
211  }
212 
213  /*
214  * Handle start up success or failure.
215  */
216  if (errors.isEmpty()) {
217  for (IngestJobPipeline ingestJobPipeline : ingestJobPipelines.values()) {
218  IngestManager.getInstance().fireDataSourceAnalysisStarted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource());
219  }
220  } else {
221  cancel(CancellationReason.INGEST_MODULES_STARTUP_FAILED);
222  }
223 
224  return errors;
225  }
226 
232  Mode getIngestMode() {
233  return ingestMode;
234  }
235 
242  return new ProgressSnapshot(true);
243  }
244 
252  public ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot) {
253  return new ProgressSnapshot(getIngestTasksSnapshot);
254  }
255 
262  List<Snapshot> getDataSourceIngestJobSnapshots() {
263  List<Snapshot> snapshots = new ArrayList<>();
264  this.ingestJobPipelines.values().stream().forEach((dataSourceJob) -> {
265  snapshots.add(dataSourceJob.getSnapshot(true));
266  });
267  return snapshots;
268  }
269 
278  @Deprecated
279  public void cancel() {
281  }
282 
291  public void cancel(CancellationReason reason) {
292  cancellationReason = reason;
293  /*
294  * Cancel the ingest pipelines for each data source. This is done in a
295  * separate thread to avoid a potential deadlock. The deadlock is
296  * possible because this method can be called in a thread that acquires
297  * the ingest manager's ingest jobs list lock and then tries to acquire
298  * the ingest pipeline stage transition lock, while an ingest thread
299  * that has acquired the stage transition lock is trying to acquire the
300  * ingest manager's ingest jobs list lock.
301  */
302  new Thread(() -> {
303  this.ingestJobPipelines.values().stream().forEach((job) -> {
304  job.cancel(reason);
305  });
306  }).start();
307  }
308 
315  return this.cancellationReason;
316  }
317 
324  public boolean isCancelled() {
325  return (CancellationReason.NOT_CANCELLED != this.cancellationReason);
326  }
327 
334  void notifyIngestPipelineShutDown(IngestJobPipeline ingestJobPipeline) {
335  IngestManager ingestManager = IngestManager.getInstance();
336  if (!ingestJobPipeline.isCancelled()) {
337  ingestManager.fireDataSourceAnalysisCompleted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource());
338  } else {
339  IngestManager.getInstance().fireDataSourceAnalysisCancelled(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource());
340  }
341  if (incompleteJobsCount.decrementAndGet() == 0) {
342  ingestManager.finishIngestJob(this);
343  }
344  }
345 
349  public final class ProgressSnapshot {
350 
351  private final List<DataSourceProcessingSnapshot> dataSourceProcessingSnapshots;
353  private boolean fileIngestRunning;
354  private Date fileIngestStartTime;
355  private final boolean jobCancelled;
357 
362  public final class DataSourceProcessingSnapshot {
363 
364  private final Snapshot snapshot;
365 
367  this.snapshot = snapshot;
368  }
369 
376  public String getDataSource() {
377  return snapshot.getDataSource();
378  }
379 
386  public boolean isCancelled() {
387  return snapshot.isCancelled();
388  }
389 
396  return snapshot.getCancellationReason();
397  }
398 
406  public List<String> getCancelledDataSourceIngestModules() {
407  return snapshot.getCancelledDataSourceIngestModules();
408  }
409 
410  }
411 
415  private ProgressSnapshot(boolean getIngestTasksSnapshot) {
416  dataSourceModule = null;
417  fileIngestRunning = false;
418  fileIngestStartTime = null;
419  dataSourceProcessingSnapshots = new ArrayList<>();
420  for (IngestJobPipeline pipeline : ingestJobPipelines.values()) {
421  Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot);
422  dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot));
423  if (null == dataSourceModule) {
424  DataSourceIngestPipeline.DataSourcePipelineModule module = snapshot.getDataSourceLevelIngestModule();
425  if (null != module) {
426  dataSourceModule = new DataSourceIngestModuleHandle(ingestJobPipelines.get(snapshot.getJobId()), module);
427  }
428  }
429  if (snapshot.getFileIngestIsRunning()) {
430  fileIngestRunning = true;
431  }
432  Date childFileIngestStartTime = snapshot.getFileIngestStartTime();
433  if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) {
434  fileIngestStartTime = childFileIngestStartTime;
435  }
436  }
437  this.jobCancelled = isCancelled();
439  }
440 
448  return this.dataSourceModule;
449  }
450 
457  public boolean fileIngestIsRunning() {
458  return this.fileIngestRunning;
459  }
460 
466  public Date fileIngestStartTime() {
467  return new Date(this.fileIngestStartTime.getTime());
468  }
469 
476  public boolean isCancelled() {
477  return this.jobCancelled;
478  }
479 
486  return this.jobCancellationReason;
487  }
488 
494  public List<DataSourceProcessingSnapshot> getDataSourceSnapshots() {
495  return Collections.unmodifiableList(this.dataSourceProcessingSnapshots);
496  }
497 
498  }
499 
505  public static class DataSourceIngestModuleHandle {
506 
507  private final IngestJobPipeline ingestJobPipeline;
508  private final DataSourceIngestPipeline.DataSourcePipelineModule module;
509  private final boolean cancelled;
510 
520  private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) {
521  this.ingestJobPipeline = ingestJobPipeline;
522  this.module = module;
523  this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled();
524  }
525 
532  public String displayName() {
533  return this.module.getDisplayName();
534  }
535 
542  public Date startTime() {
543  return this.module.getProcessingStartTime();
544  }
545 
552  public boolean isCancelled() {
553  return this.cancelled;
554  }
555 
561  public void cancel() {
573  if (this.ingestJobPipeline.getCurrentDataSourceIngestModule() == this.module) {
574  this.ingestJobPipeline.cancelCurrentDataSourceIngestModule();
575  }
576  }
577 
578  }
579 
580 }
static synchronized IngestManager getInstance()
final AtomicInteger incompleteJobsCount
Definition: IngestJob.java:81
void cancel(CancellationReason reason)
Definition: IngestJob.java:291
DataSourceIngestModuleHandle runningDataSourceIngestModule()
Definition: IngestJob.java:447
List< DataSourceProcessingSnapshot > getDataSourceSnapshots()
Definition: IngestJob.java:494
final IngestJob.CancellationReason jobCancellationReason
Definition: IngestJob.java:356
CancellationReason getCancellationReason()
Definition: IngestJob.java:314
final Map< Long, IngestJobPipeline > ingestJobPipelines
Definition: IngestJob.java:80
DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module)
Definition: IngestJob.java:520
final List< Content > dataSources
Definition: IngestJob.java:77
final List< AbstractFile > files
Definition: IngestJob.java:78
final DataSourceIngestPipeline.DataSourcePipelineModule module
Definition: IngestJob.java:508
static final AtomicLong nextId
Definition: IngestJob.java:75
ProgressSnapshot(boolean getIngestTasksSnapshot)
Definition: IngestJob.java:415
volatile CancellationReason cancellationReason
Definition: IngestJob.java:83
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot)
Definition: IngestJob.java:252
final IngestJobSettings settings
Definition: IngestJob.java:82
final List< DataSourceProcessingSnapshot > dataSourceProcessingSnapshots
Definition: IngestJob.java:351

Copyright © 2012-2021 Basis Technology. Generated on: Thu Sep 30 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.