Autopsy  4.12.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
CommandLineIngestManager.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2019-2019 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.commandlineingest;
20 
21 import java.beans.PropertyChangeEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.File;
24 import java.io.FilenameFilter;
25 import java.nio.file.Path;
26 import java.nio.file.Paths;
27 import java.util.List;
28 import java.util.UUID;
29 import java.util.Collection;
30 import java.util.EnumSet;
31 import java.util.Iterator;
32 import java.util.Set;
33 import java.util.logging.Level;
34 import org.netbeans.spi.sendopts.OptionProcessor;
35 import org.openide.LifecycleManager;
36 import org.openide.util.Lookup;
60 import org.sleuthkit.datamodel.Content;
61 
68 
69  private static final Logger LOGGER = Logger.getLogger(CommandLineIngestManager.class.getName());
71  private Path rootOutputDirectory;
72 
74  }
75 
76  public void start() {
77  new Thread(new JobProcessingTask()).start();
78  }
79 
80  public void stop() {
81  try {
82  // close current case if there is one open
84  } catch (CaseActionException ex) {
85  LOGGER.log(Level.WARNING, "Unable to close the case while shutting down command line ingest manager", ex); //NON-NLS
86  }
87 
88  // shut down Autopsy
89  LifecycleManager.getDefault().exit();
90  }
91 
92  private final class JobProcessingTask implements Runnable {
93 
94  private final Object ingestLock;
95 
96  private JobProcessingTask() {
97  ingestLock = new Object();
98  try {
100  LOGGER.log(Level.INFO, "Set running with desktop GUI runtime property to false");
102  LOGGER.log(Level.SEVERE, "Failed to set running with desktop GUI runtime property to false", ex);
103  }
104  }
105 
106  @Override
107  public void run() {
108  LOGGER.log(Level.INFO, "Job processing task started");
109 
110  try {
111  // read command line inputs
112  LOGGER.log(Level.INFO, "Autopsy is running from command line"); //NON-NLS
113  String dataSourcePath = "";
114  String baseCaseName = "";
115 
116  // first look up all OptionProcessors and get input data from CommandLineOptionProcessor
117  Collection<? extends OptionProcessor> optionProcessors = Lookup.getDefault().lookupAll(OptionProcessor.class);
118  Iterator<? extends OptionProcessor> optionsIterator = optionProcessors.iterator();
119  while (optionsIterator.hasNext()) {
120  // find CommandLineOptionProcessor
121  OptionProcessor processor = optionsIterator.next();
122  if (processor instanceof CommandLineOptionProcessor) {
123  // check if we are running from command line
124  dataSourcePath = ((CommandLineOptionProcessor) processor).getPathToDataSource();
125  baseCaseName = ((CommandLineOptionProcessor) processor).getBaseCaseName();
126  }
127  }
128 
129  LOGGER.log(Level.INFO, "Data source path = {0}", dataSourcePath); //NON-NLS
130  LOGGER.log(Level.INFO, "Case name = {0}", baseCaseName); //NON-NLS
131  System.out.println("Data source path = " + dataSourcePath);
132  System.out.println("Case name = " + baseCaseName);
133 
134  // verify inputs
135  if (dataSourcePath.isEmpty()) {
136  LOGGER.log(Level.SEVERE, "Data source path not specified");
137  System.out.println("Data source path not specified");
138  return;
139  }
140 
141  if (baseCaseName.isEmpty()) {
142  LOGGER.log(Level.SEVERE, "Case name not specified");
143  System.out.println("Case name not specified");
144  return;
145  }
146 
147  if (!(new File(dataSourcePath).exists())) {
148  LOGGER.log(Level.SEVERE, "Data source file not found {0}", dataSourcePath);
149  System.out.println("Data source file not found " + dataSourcePath);
150  return;
151  }
152 
153  // read options panel configuration
154  String rootOutputDir = UserPreferences.getCommandLineModeResultsFolder();
155  LOGGER.log(Level.INFO, "Output directory = {0}", rootOutputDir); //NON-NLS
156  System.out.println("Output directory = " + rootOutputDir);
157 
158  if (rootOutputDir.isEmpty()) {
159  LOGGER.log(Level.SEVERE, "Output directory not specified, please configure Command Line Options Panel (in Tools -> Options)");
160  System.out.println("Output directory not specified, please configure Command Line Options Panel (in Tools -> Options)");
161  return;
162  }
163 
164  if (!(new File(rootOutputDir).exists())) {
165  LOGGER.log(Level.SEVERE, "The output directory doesn't exist {0}", rootOutputDir);
166  System.out.println("The output directory doesn't exist " + rootOutputDir);
167  return;
168  }
169  rootOutputDirectory = Paths.get(rootOutputDir);
170 
171  // open case
172  Case caseForJob;
173  try {
174  caseForJob = openCase(baseCaseName);
175  } catch (CaseActionException ex) {
176  LOGGER.log(Level.SEVERE, "Error creating or opening case " + baseCaseName, ex);
177  System.out.println("Error creating or opening case " + baseCaseName);
178  return;
179  }
180 
181  if (caseForJob == null) {
182  LOGGER.log(Level.SEVERE, "Error creating or opening case {0}", baseCaseName);
183  System.out.println("Error creating or opening case " + baseCaseName);
184  return;
185  }
186 
187  AutoIngestDataSource dataSource = new AutoIngestDataSource("", Paths.get(dataSourcePath));
188  try {
189  // run data source processor
190  runDataSourceProcessor(caseForJob, dataSource);
191 
192  // run ingest manager
193  analyze(dataSource);
194 
195  // generate CASE-UCO report
196  Long selectedDataSourceId = getDataSourceId(dataSource);
197  Path reportFolderPath = Paths.get(caseForJob.getReportDirectory(), "CASE-UCO", "Data_Source_ID_" + selectedDataSourceId.toString() + "_" + TimeStampUtils.createTimeStamp(), ReportCaseUco.getReportFileName()); //NON_NLS
198  ReportProgressPanel progressPanel = new ReportProgressPanel("CASE_UCO", rootOutputDir); // dummy progress panel
199  CaseUcoFormatExporter.generateReport(selectedDataSourceId, reportFolderPath.toString(), progressPanel);
201  LOGGER.log(Level.SEVERE, "Unable to ingest data source " + dataSourcePath + ". Exiting...", ex);
202  System.out.println("Unable to ingest data source " + dataSourcePath + ". Exiting...");
203  } catch (Throwable ex) {
204  /*
205  * Unexpected runtime exceptions firewall. This task is
206  * designed to be able to be run in an executor service
207  * thread pool without calling get() on the task's
208  * Future<Void>, so this ensures that such errors get
209  * logged.
210  */
211  LOGGER.log(Level.SEVERE, "Unexpected error while ingesting data source " + dataSourcePath, ex);
212  System.out.println("Unexpected error while ingesting data source " + dataSourcePath + ". Exiting...");
213 
214  } finally {
215  try {
217  } catch (CaseActionException ex) {
218  LOGGER.log(Level.WARNING, "Exception while closing case", ex);
219  System.out.println("Exception while closing case");
220  }
221  }
222 
223  } finally {
224  LOGGER.log(Level.INFO, "Job processing task finished");
225  System.out.println("Job processing task finished");
226 
227  // shut down Autopsy
228  stop();
229  }
230  }
231 
240  private Long getDataSourceId(AutoIngestDataSource dataSource) {
241  Content content = dataSource.getContent().get(0);
242  return content.getId();
243  }
244 
245  private Case openCase(String baseCaseName) throws CaseActionException {
246 
247  LOGGER.log(Level.INFO, "Opening case {0}", baseCaseName);
248 
249  Path caseDirectoryPath = findCaseDirectory(rootOutputDirectory, baseCaseName);
250  if (null != caseDirectoryPath) {
251  // found an existing case directory for same case name. the input case name must be unique. Exit.
252  LOGGER.log(Level.SEVERE, "Case {0} already exists. Case name must be unique. Exiting", baseCaseName);
253  throw new CaseActionException("Case " + baseCaseName + " already exists. Case name must be unique. Exiting");
254  } else {
255  caseDirectoryPath = createCaseFolderPath(rootOutputDirectory, baseCaseName);
256 
257  // Create the case directory
258  Case.createCaseDirectory(caseDirectoryPath.toString(), Case.CaseType.SINGLE_USER_CASE);
259 
260  CaseDetails caseDetails = new CaseDetails(baseCaseName);
261  Case.createAsCurrentCase(Case.CaseType.SINGLE_USER_CASE, caseDirectoryPath.toString(), caseDetails);
262  }
263 
264  Case caseForJob = Case.getCurrentCase();
265  LOGGER.log(Level.INFO, "Opened case {0}", caseForJob.getName());
266  return caseForJob;
267  }
268 
284 
285  LOGGER.log(Level.INFO, "Adding data source {0} ", dataSource.getPath().toString());
286 
287  // Get an ordered list of data source processors to try
288  List<AutoIngestDataSourceProcessor> validDataSourceProcessors;
289  try {
290  validDataSourceProcessors = DataSourceProcessorUtility.getOrderedListOfDataSourceProcessors(dataSource.getPath());
292  LOGGER.log(Level.SEVERE, "Exception while determining best data source processor for {0}", dataSource.getPath());
293  // rethrow the exception.
294  throw ex;
295  }
296 
297  // did we find a data source processor that can process the data source
298  if (validDataSourceProcessors.isEmpty()) {
299  // This should never happen. We should add all unsupported data sources as logical files.
300  LOGGER.log(Level.SEVERE, "Unsupported data source {0}", dataSource.getPath()); // NON-NLS
301  return;
302  }
303 
305  synchronized (ingestLock) {
306  // Try each DSP in decreasing order of confidence
307  for (AutoIngestDataSourceProcessor selectedProcessor : validDataSourceProcessors) {
308  UUID taskId = UUID.randomUUID();
309  caseForJob.notifyAddingDataSource(taskId);
310  DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId, ingestLock);
311  caseForJob.notifyAddingDataSource(taskId);
312  LOGGER.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{dataSource.getPath(), selectedProcessor.getDataSourceType()});
313  selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack);
314  ingestLock.wait();
315 
316  // at this point we got the content object(s) from the current DSP.
317  // check whether the data source was processed successfully
318  if ((dataSource.getResultDataSourceProcessorResultCode() == CRITICAL_ERRORS)
319  || dataSource.getContent().isEmpty()) {
320  // move onto the the next DSP that can process this data source
321  logDataSourceProcessorResult(dataSource);
322  continue;
323  }
324 
325  logDataSourceProcessorResult(dataSource);
326  return;
327  }
328  // If we get to this point, none of the processors were successful
329  LOGGER.log(Level.SEVERE, "All data source processors failed to process {0}", dataSource.getPath());
330  // Throw an exception. It will get caught & handled upstream and will result in AIM auto-pause.
331  throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Failed to process " + dataSource.getPath() + " with all data source processors");
332  }
333  }
334 
342 
344  if (null != resultCode) {
345  switch (resultCode) {
346  case NO_ERRORS:
347  LOGGER.log(Level.INFO, "Added data source to case");
348  if (dataSource.getContent().isEmpty()) {
349  LOGGER.log(Level.SEVERE, "Data source failed to produce content");
350  }
351  break;
352 
353  case NONCRITICAL_ERRORS:
354  for (String errorMessage : dataSource.getDataSourceProcessorErrorMessages()) {
355  LOGGER.log(Level.WARNING, "Non-critical error running data source processor for {0}: {1}", new Object[]{dataSource.getPath(), errorMessage});
356  }
357  LOGGER.log(Level.INFO, "Added data source to case");
358  if (dataSource.getContent().isEmpty()) {
359  LOGGER.log(Level.SEVERE, "Data source failed to produce content");
360  }
361  break;
362 
363  case CRITICAL_ERRORS:
364  for (String errorMessage : dataSource.getDataSourceProcessorErrorMessages()) {
365  LOGGER.log(Level.SEVERE, "Critical error running data source processor for {0}: {1}", new Object[]{dataSource.getPath(), errorMessage});
366  }
367  LOGGER.log(Level.SEVERE, "Failed to add data source to case");
368  break;
369  }
370  } else {
371  LOGGER.log(Level.WARNING, "No result code for data source processor for {0}", dataSource.getPath());
372  }
373  }
374 
388  private void analyze(AutoIngestDataSource dataSource) throws AnalysisStartupException, InterruptedException {
389 
390  LOGGER.log(Level.INFO, "Starting ingest modules analysis for {0} ", dataSource.getPath());
391  IngestJobEventListener ingestJobEventListener = new IngestJobEventListener();
393  try {
394  synchronized (ingestLock) {
396  List<String> settingsWarnings = ingestJobSettings.getWarnings();
397  if (settingsWarnings.isEmpty()) {
398  IngestJobStartResult ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings);
399  IngestJob ingestJob = ingestJobStartResult.getJob();
400  if (null != ingestJob) {
401  /*
402  * Block until notified by the ingest job event
403  * listener or until interrupted because auto ingest
404  * is shutting down.
405  */
406  ingestLock.wait();
407  LOGGER.log(Level.INFO, "Finished ingest modules analysis for {0} ", dataSource.getPath());
408  IngestJob.ProgressSnapshot jobSnapshot = ingestJob.getSnapshot();
409  for (IngestJob.ProgressSnapshot.DataSourceProcessingSnapshot snapshot : jobSnapshot.getDataSourceSnapshots()) {
410  if (!snapshot.isCancelled()) {
411  List<String> cancelledModules = snapshot.getCancelledDataSourceIngestModules();
412  if (!cancelledModules.isEmpty()) {
413  LOGGER.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", dataSource.getPath()));
414  for (String module : snapshot.getCancelledDataSourceIngestModules()) {
415  LOGGER.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, dataSource.getPath()));
416  }
417  }
418  LOGGER.log(Level.INFO, "Analysis of data source completed");
419  } else {
420  LOGGER.log(Level.WARNING, "Analysis of data source cancelled");
421  IngestJob.CancellationReason cancellationReason = snapshot.getCancellationReason();
422  if (IngestJob.CancellationReason.NOT_CANCELLED != cancellationReason && IngestJob.CancellationReason.USER_CANCELLED != cancellationReason) {
423  throw new AnalysisStartupException(String.format("Analysis cancelled due to %s for %s", cancellationReason.getDisplayName(), dataSource.getPath()));
424  }
425  }
426  }
427  } else if (!ingestJobStartResult.getModuleErrors().isEmpty()) {
428  for (IngestModuleError error : ingestJobStartResult.getModuleErrors()) {
429  LOGGER.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), dataSource.getPath()), error.getThrowable());
430  }
431  LOGGER.log(Level.SEVERE, "Failed to analyze data source due to ingest job startup error");
432  throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", dataSource.getPath()));
433  } else {
434  LOGGER.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", dataSource.getPath()), ingestJobStartResult.getStartupException());
435  throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException());
436  }
437  } else {
438  for (String warning : settingsWarnings) {
439  LOGGER.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{dataSource.getPath(), warning});
440  }
441  LOGGER.log(Level.SEVERE, "Failed to analyze data source due to settings errors");
442  throw new AnalysisStartupException("Error(s) in ingest job settings");
443  }
444  }
445  } finally {
446  IngestManager.getInstance().removeIngestJobEventListener(ingestJobEventListener);
447  }
448  }
449 
459  Path createCaseFolderPath(Path caseFoldersPath, String caseName) {
460  String folderName = caseName + "_" + TimeStampUtils.createTimeStamp();
461  return Paths.get(caseFoldersPath.toString(), folderName);
462  }
463 
474  Path findCaseDirectory(Path folderToSearch, String caseName) {
475  File searchFolder = new File(folderToSearch.toString());
476  if (!searchFolder.isDirectory()) {
477  return null;
478  }
479  Path caseFolderPath = null;
480  String[] candidateFolders = searchFolder.list(new CaseFolderFilter(caseName));
481  long mostRecentModified = 0;
482  for (String candidateFolder : candidateFolders) {
483  File file = new File(candidateFolder);
484  if (file.lastModified() >= mostRecentModified) {
485  mostRecentModified = file.lastModified();
486  caseFolderPath = Paths.get(folderToSearch.toString(), file.getPath());
487  }
488  }
489  return caseFolderPath;
490  }
491 
501  private class IngestJobEventListener implements PropertyChangeListener {
502 
510  @Override
511  public void propertyChange(PropertyChangeEvent event) {
512  if (AutopsyEvent.SourceType.LOCAL == ((AutopsyEvent) event).getSourceType()) {
513  String eventType = event.getPropertyName();
514  if (eventType.equals(IngestManager.IngestJobEvent.COMPLETED.toString()) || eventType.equals(IngestManager.IngestJobEvent.CANCELLED.toString())) {
515  synchronized (ingestLock) {
516  ingestLock.notify();
517  }
518  }
519  }
520  }
521  };
522 
529 
535  @Override
536  public void setIndeterminate(final boolean indeterminate) {
537  }
538 
544  @Override
545  public void setProgress(final int progress) {
546  }
547 
553  @Override
554  public void setProgressText(final String text) {
555  }
556  }
557 
563  private final class AnalysisStartupException extends Exception {
564 
565  private static final long serialVersionUID = 1L;
566 
567  private AnalysisStartupException(String message) {
568  super(message);
569  }
570 
571  private AnalysisStartupException(String message, Throwable cause) {
572  super(message, cause);
573  }
574  }
575  }
576 
577  private static class CaseFolderFilter implements FilenameFilter {
578 
579  private final String caseName;
580  private final static String CASE_METADATA_EXT = CaseMetadata.getFileExtension();
581 
582  CaseFolderFilter(String caseName) {
583  this.caseName = caseName;
584  }
585 
586  @Override
587  public boolean accept(File folder, String fileName) {
588  File file = new File(folder, fileName);
589  if (fileName.length() > TimeStampUtils.getTimeStampLength() && file.isDirectory()) {
590  if (TimeStampUtils.endsWithTimeStamp(fileName)) {
591  if (null != caseName) {
592  String fileNamePrefix = fileName.substring(0, fileName.length() - TimeStampUtils.getTimeStampLength());
593  if (fileNamePrefix.equals(caseName)) {
594  return hasCaseMetadataFile(file);
595  }
596  } else {
597  return hasCaseMetadataFile(file);
598  }
599  }
600  }
601  return false;
602  }
603 
612  private static boolean hasCaseMetadataFile(File folder) {
613  for (File file : folder.listFiles()) {
614  if (file.getName().toLowerCase().endsWith(CASE_METADATA_EXT) && file.isFile()) {
615  return true;
616  }
617  }
618  return false;
619  }
620  }
621 
622 }
static List< AutoIngestDataSourceProcessor > getOrderedListOfDataSourceProcessors(Path dataSourcePath)
static synchronized IngestManager getInstance()
static void generateReport(Long selectedDataSourceId, String reportOutputPath, ReportProgressPanel progressPanel)
static void createCaseDirectory(String caseDirPath, CaseType caseType)
Definition: Case.java:884
IngestJobStartResult beginIngestJob(Collection< Content > dataSources, IngestJobSettings settings)
synchronized DataSourceProcessorResult getResultDataSourceProcessorResultCode()
static final Set< IngestManager.IngestJobEvent > INGEST_JOB_EVENTS_OF_INTEREST
static boolean endsWithTimeStamp(String inputString)
void removeIngestJobEventListener(final PropertyChangeListener listener)
void addIngestJobEventListener(final PropertyChangeListener listener)
static synchronized void setRunningWithGUI(boolean runningWithGUI)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static void createAsCurrentCase(String caseDir, String caseDisplayName, String caseNumber, String examiner, CaseType caseType)
Definition: Case.java:567
IngestManager.IngestManagerException getStartupException()

Copyright © 2012-2018 Basis Technology. Generated on: Wed Sep 18 2019
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.