Autopsy  4.5.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
PhotoRecCarverFileIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2016 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.photoreccarver;
20 
21 import java.io.File;
22 import java.io.IOException;
23 import java.lang.ProcessBuilder.Redirect;
24 import java.nio.file.DirectoryStream;
25 import java.nio.file.FileAlreadyExistsException;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.Paths;
29 import java.text.DateFormat;
30 import java.text.SimpleDateFormat;
31 import java.util.ArrayList;
32 import java.util.Date;
33 import java.util.HashMap;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.concurrent.ConcurrentHashMap;
37 import java.util.concurrent.atomic.AtomicLong;
38 import java.util.logging.Level;
39 import org.openide.modules.InstalledFileLocator;
40 import org.openide.util.Exceptions;
41 import org.openide.util.NbBundle;
60 import org.sleuthkit.datamodel.AbstractFile;
61 import org.sleuthkit.datamodel.LayoutFile;
62 import org.sleuthkit.datamodel.TskData;
63 
68 @NbBundle.Messages({
69  "PhotoRecIngestModule.PermissionsNotSufficient=Insufficient permissions accessing",
70  "PhotoRecIngestModule.PermissionsNotSufficientSeeReference=See 'Shared Drive Authentication' in Autopsy help.",
71  "# {0} - output directory name", "cannotCreateOutputDir.message=Unable to create output directory: {0}.",
72  "unallocatedSpaceProcessingSettingsError.message=The selected file ingest filter ignores unallocated space. This module carves unallocated space. Please choose a filter which does not ignore unallocated space or disable this module.",
73  "unsupportedOS.message=PhotoRec module is supported on Windows platforms only.",
74  "missingExecutable.message=Unable to locate PhotoRec executable.",
75  "cannotRunExecutable.message=Unable to execute PhotoRec.",
76  "PhotoRecIngestModule.nonHostnameUNCPathUsed=PhotoRec cannot operate with a UNC path containing IP addresses."
77 })
78 final class PhotoRecCarverFileIngestModule implements FileIngestModule {
79 
80  private static final String PHOTOREC_DIRECTORY = "photorec_exec"; //NON-NLS
81  private static final String PHOTOREC_EXECUTABLE = "photorec_win.exe"; //NON-NLS
82  private static final String PHOTOREC_LINUX_EXECUTABLE = "photorec";
83  private static final String PHOTOREC_RESULTS_BASE = "results"; //NON-NLS
84  private static final String PHOTOREC_RESULTS_EXTENDED = "results.1"; //NON-NLS
85  private static final String PHOTOREC_REPORT = "report.xml"; //NON-NLS
86  private static final String LOG_FILE = "run_log.txt"; //NON-NLS
87  private static final String TEMP_DIR_NAME = "temp"; // NON-NLS
88  private static final String SEP = System.getProperty("line.separator");
89  private static final Logger logger = Logger.getLogger(PhotoRecCarverFileIngestModule.class.getName());
90  private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs = new HashMap<>();
91  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
92  private static final Map<Long, WorkingPaths> pathsByJob = new ConcurrentHashMap<>();
93  private IngestJobContext context;
94  private Path rootOutputDirPath;
95  private File executableFile;
96  private IngestServices services;
97  private UNCPathUtilities uncPathUtilities = new UNCPathUtilities();
98  private long jobId;
99 
100  private static class IngestJobTotals {
101 
102  private AtomicLong totalItemsRecovered = new AtomicLong(0);
103  private AtomicLong totalItemsWithErrors = new AtomicLong(0);
104  private AtomicLong totalWritetime = new AtomicLong(0);
105  private AtomicLong totalParsetime = new AtomicLong(0);
106  }
107 
108  private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) {
109  IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId);
110  if (totals == null) {
111  totals = new PhotoRecCarverFileIngestModule.IngestJobTotals();
112  totalsForIngestJobs.put(ingestJobId, totals);
113  }
114  return totals;
115  }
116 
117  private static synchronized void initTotalsForIngestJob(long ingestJobId) {
118  IngestJobTotals totals = new PhotoRecCarverFileIngestModule.IngestJobTotals();
119  totalsForIngestJobs.put(ingestJobId, totals);
120  }
121 
125  @Override
126  public void startUp(IngestJobContext context) throws IngestModule.IngestModuleException {
127  this.context = context;
128  this.services = IngestServices.getInstance();
129  this.jobId = this.context.getJobId();
130 
131  // If the global unallocated space processing setting and the module
132  // process unallocated space only setting are not in sych, throw an
133  // exception. Although the result would not be incorrect, it would be
134  // unfortunate for the user to get an accidental no-op for this module.
135  if (!this.context.processingUnallocatedSpace()) {
136  throw new IngestModule.IngestModuleException(Bundle.unallocatedSpaceProcessingSettingsError_message());
137  }
138 
139  this.rootOutputDirPath = createModuleOutputDirectoryForCase();
140 
141  //Set photorec executable directory based on operating system.
142  executableFile = locateExecutable();
143 
144  if (PhotoRecCarverFileIngestModule.refCounter.incrementAndGet(this.jobId) == 1) {
145  try {
146  // The first instance creates an output subdirectory with a date and time stamp
147  DateFormat dateFormat = new SimpleDateFormat("MM-dd-yyyy-HH-mm-ss-SSSS"); // NON-NLS
148  Date date = new Date();
149  String folder = this.context.getDataSource().getId() + "_" + dateFormat.format(date);
150  Path outputDirPath = Paths.get(this.rootOutputDirPath.toAbsolutePath().toString(), folder);
151  Files.createDirectories(outputDirPath);
152 
153  // A temp subdirectory is also created as a location for writing unallocated space files to disk.
154  Path tempDirPath = Paths.get(outputDirPath.toString(), PhotoRecCarverFileIngestModule.TEMP_DIR_NAME);
155  Files.createDirectory(tempDirPath);
156 
157  // Save the directories for the current job.
158  PhotoRecCarverFileIngestModule.pathsByJob.put(this.jobId, new WorkingPaths(outputDirPath, tempDirPath));
159 
160  // Initialize job totals
161  initTotalsForIngestJob(jobId);
162  } catch (SecurityException | IOException | UnsupportedOperationException ex) {
163  throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
164  }
165  }
166  }
167 
171  @Override
172  public IngestModule.ProcessResult process(AbstractFile file) {
173  // Skip everything except unallocated space files.
174  if (file.getType() != TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) {
175  return IngestModule.ProcessResult.OK;
176  }
177 
178  // Safely get a reference to the totalsForIngestJobs object
179  IngestJobTotals totals = getTotalsForIngestJobs(jobId);
180 
181  Path tempFilePath = null;
182  try {
183  // Verify initialization succeeded.
184  if (null == this.executableFile) {
185  logger.log(Level.SEVERE, "PhotoRec carver called after failed start up"); // NON-NLS
186  return IngestModule.ProcessResult.ERROR;
187  }
188 
189  // Check that we have roughly enough disk space left to complete the operation
190  // Some network drives always return -1 for free disk space.
191  // In this case, expect enough space and move on.
192  long freeDiskSpace = IngestServices.getInstance().getFreeDiskSpace();
193  if ((freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && ((file.getSize() * 1.2) > freeDiskSpace)) {
194  logger.log(Level.SEVERE, "PhotoRec error processing {0} with {1} Not enough space on primary disk to save unallocated space.", // NON-NLS
195  new Object[]{file.getName(), PhotoRecCarverIngestModuleFactory.getModuleName()}); // NON-NLS
196  MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.UnableToCarve", file.getName()),
197  NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.NotEnoughDiskSpace"));
198  return IngestModule.ProcessResult.ERROR;
199  }
200  if (this.context.fileIngestIsCancelled() == true) {
201  // if it was cancelled by the user, result is OK
202  logger.log(Level.INFO, "PhotoRec cancelled by user"); // NON-NLS
203  MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.cancelledByUser"));
204  return IngestModule.ProcessResult.OK;
205  }
206 
207  // Write the file to disk.
208  long writestart = System.currentTimeMillis();
209  WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.get(this.jobId);
210  tempFilePath = Paths.get(paths.getTempDirPath().toString(), file.getName());
211  ContentUtils.writeToFile(file, tempFilePath.toFile(), context::fileIngestIsCancelled);
212 
213  if (this.context.fileIngestIsCancelled() == true) {
214  // if it was cancelled by the user, result is OK
215  logger.log(Level.INFO, "PhotoRec cancelled by user"); // NON-NLS
216  MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.cancelledByUser"));
217  return IngestModule.ProcessResult.OK;
218  }
219 
220  // Create a subdirectory for this file.
221  Path outputDirPath = Paths.get(paths.getOutputDirPath().toString(), file.getName());
222  Files.createDirectory(outputDirPath);
223  File log = new File(Paths.get(outputDirPath.toString(), LOG_FILE).toString()); //NON-NLS
224 
225  // Scan the file with Unallocated Carver.
226  ProcessBuilder processAndSettings = new ProcessBuilder(
227  executableFile.toString(),
228  "/d", // NON-NLS
229  outputDirPath.toAbsolutePath().toString() + File.separator + PHOTOREC_RESULTS_BASE,
230  "/cmd", // NON-NLS
231  tempFilePath.toFile().toString(),
232  "search"); // NON-NLS
233 
234  // Add environment variable to force PhotoRec to run with the same permissions Autopsy uses
235  processAndSettings.environment().put("__COMPAT_LAYER", "RunAsInvoker"); //NON-NLS
236  processAndSettings.redirectErrorStream(true);
237  processAndSettings.redirectOutput(Redirect.appendTo(log));
238 
239  FileIngestModuleProcessTerminator terminator = new FileIngestModuleProcessTerminator(this.context, true);
240  int exitValue = ExecUtil.execute(processAndSettings, terminator);
241 
242  if (this.context.fileIngestIsCancelled() == true) {
243  // if it was cancelled by the user, result is OK
244  cleanup(outputDirPath, tempFilePath);
245  logger.log(Level.INFO, "PhotoRec cancelled by user"); // NON-NLS
246  MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.cancelledByUser"));
247  return IngestModule.ProcessResult.OK;
248  } else if (terminator.getTerminationCode() == ProcTerminationCode.TIME_OUT) {
249  cleanup(outputDirPath, tempFilePath);
250  String msg = NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.processTerminated") + file.getName(); // NON-NLS
251  MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "PhotoRecIngestModule.moduleError"), msg); // NON-NLS
252  logger.log(Level.SEVERE, msg);
253  return IngestModule.ProcessResult.ERROR;
254  } else if (0 != exitValue) {
255  // if it failed or was cancelled by timeout, result is ERROR
256  cleanup(outputDirPath, tempFilePath);
257  totals.totalItemsWithErrors.incrementAndGet();
258  logger.log(Level.SEVERE, "PhotoRec carver returned error exit value = {0} when scanning {1}", // NON-NLS
259  new Object[]{exitValue, file.getName()}); // NON-NLS
260  MessageNotifyUtil.Notify.error(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.error.exitValue", // NON-NLS
261  new Object[]{exitValue, file.getName()}));
262  return IngestModule.ProcessResult.ERROR;
263  }
264 
265  // Move carver log file to avoid placement into Autopsy results. PhotoRec appends ".1" to the folder name.
266  java.io.File oldAuditFile = new java.io.File(Paths.get(outputDirPath.toString(), PHOTOREC_RESULTS_EXTENDED, PHOTOREC_REPORT).toString()); //NON-NLS
267  java.io.File newAuditFile = new java.io.File(Paths.get(outputDirPath.toString(), PHOTOREC_REPORT).toString()); //NON-NLS
268  oldAuditFile.renameTo(newAuditFile);
269 
270  if (this.context.fileIngestIsCancelled() == true) {
271  // if it was cancelled by the user, result is OK
272  logger.log(Level.INFO, "PhotoRec cancelled by user"); // NON-NLS
273  MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.cancelledByUser"));
274  return IngestModule.ProcessResult.OK;
275  }
276  Path pathToRemove = Paths.get(outputDirPath.toAbsolutePath().toString());
277  try (DirectoryStream<Path> stream = Files.newDirectoryStream(pathToRemove)) {
278  for (Path entry : stream) {
279  if (Files.isDirectory(entry)) {
280  FileUtil.deleteDir(new File(entry.toString()));
281  }
282  }
283  }
284  long writedelta = (System.currentTimeMillis() - writestart);
285  totals.totalWritetime.addAndGet(writedelta);
286 
287  // Now that we've cleaned up the folders and data files, parse the xml output file to add carved items into the database
288  long calcstart = System.currentTimeMillis();
289  PhotoRecCarverOutputParser parser = new PhotoRecCarverOutputParser(outputDirPath);
290  if (this.context.fileIngestIsCancelled() == true) {
291  // if it was cancelled by the user, result is OK
292  logger.log(Level.INFO, "PhotoRec cancelled by user"); // NON-NLS
293  MessageNotifyUtil.Notify.info(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.cancelledByUser"));
294  return IngestModule.ProcessResult.OK;
295  }
296  List<LayoutFile> carvedItems = parser.parse(newAuditFile, file, context);
297  long calcdelta = (System.currentTimeMillis() - calcstart);
298  totals.totalParsetime.addAndGet(calcdelta);
299  if (carvedItems != null && !carvedItems.isEmpty()) { // if there were any results from carving, add the unallocated carving event to the reports list.
300  totals.totalItemsRecovered.addAndGet(carvedItems.size());
301  context.addFilesToJob(new ArrayList<>(carvedItems));
302  services.fireModuleContentEvent(new ModuleContentEvent(carvedItems.get(0))); // fire an event to update the tree
303  }
304  } catch (IOException ex) {
305  totals.totalItemsWithErrors.incrementAndGet();
306  logger.log(Level.SEVERE, "Error processing " + file.getName() + " with PhotoRec carver", ex); // NON-NLS
307  MessageNotifyUtil.Notify.error(PhotoRecCarverIngestModuleFactory.getModuleName(), NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.error.msg", file.getName()));
308  return IngestModule.ProcessResult.ERROR;
309  } finally {
310  if (null != tempFilePath && Files.exists(tempFilePath)) {
311  // Get rid of the unallocated space file.
312  tempFilePath.toFile().delete();
313  }
314  }
315  return IngestModule.ProcessResult.OK;
316 
317  }
318 
319  private void cleanup(Path outputDirPath, Path tempFilePath) {
320  // cleanup the output path
321  FileUtil.deleteDir(new File(outputDirPath.toString()));
322  if (null != tempFilePath && Files.exists(tempFilePath)) {
323  tempFilePath.toFile().delete();
324  }
325  }
326 
327  private static synchronized void postSummary(long jobId) {
328  IngestJobTotals jobTotals = totalsForIngestJobs.remove(jobId);
329 
330  StringBuilder detailsSb = new StringBuilder();
331  //details
332  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
333 
334  detailsSb.append("<tr><td>") //NON-NLS
335  .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.complete.numberOfCarved"))
336  .append("</td>"); //NON-NLS
337  detailsSb.append("<td>").append(jobTotals.totalItemsRecovered.get()).append("</td></tr>"); //NON-NLS
338 
339  detailsSb.append("<tr><td>") //NON-NLS
340  .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.complete.numberOfErrors"))
341  .append("</td>"); //NON-NLS
342  detailsSb.append("<td>").append(jobTotals.totalItemsWithErrors.get()).append("</td></tr>"); //NON-NLS
343 
344  detailsSb.append("<tr><td>") //NON-NLS
345  .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.complete.totalWritetime"))
346  .append("</td><td>").append(jobTotals.totalWritetime.get()).append("</td></tr>\n"); //NON-NLS
347  detailsSb.append("<tr><td>") //NON-NLS
348  .append(NbBundle.getMessage(PhotoRecCarverFileIngestModule.class, "PhotoRecIngestModule.complete.totalParsetime"))
349  .append("</td><td>").append(jobTotals.totalParsetime.get()).append("</td></tr>\n"); //NON-NLS
350  detailsSb.append("</table>"); //NON-NLS
351 
352  IngestServices.getInstance().postMessage(IngestMessage.createMessage(
353  IngestMessage.MessageType.INFO,
354  PhotoRecCarverIngestModuleFactory.getModuleName(),
355  NbBundle.getMessage(PhotoRecCarverFileIngestModule.class,
356  "PhotoRecIngestModule.complete.photoRecResults"),
357  detailsSb.toString()));
358 
359  }
360 
364  @Override
365  public void shutDown() {
366  if (this.context != null && refCounter.decrementAndGet(this.jobId) == 0) {
367  try {
368  // The last instance of this module for an ingest job cleans out
369  // the working paths map entry for the job and deletes the temp dir.
370  WorkingPaths paths = PhotoRecCarverFileIngestModule.pathsByJob.remove(this.jobId);
371  FileUtil.deleteDir(new File(paths.getTempDirPath().toString()));
372  postSummary(jobId);
373  } catch (SecurityException ex) {
374  logger.log(Level.SEVERE, "Error shutting down PhotoRec carver module", ex); // NON-NLS
375  }
376  }
377  }
378 
379  private static final class WorkingPaths {
380 
381  private final Path outputDirPath;
382  private final Path tempDirPath;
383 
384  WorkingPaths(Path outputDirPath, Path tempDirPath) {
385  this.outputDirPath = outputDirPath;
386  this.tempDirPath = tempDirPath;
387  }
388 
389  Path getOutputDirPath() {
390  return this.outputDirPath;
391  }
392 
393  Path getTempDirPath() {
394  return this.tempDirPath;
395  }
396  }
397 
406  synchronized Path createModuleOutputDirectoryForCase() throws IngestModule.IngestModuleException {
407  Path path = Paths.get(Case.getCurrentCase().getModuleDirectory(), PhotoRecCarverIngestModuleFactory.getModuleName());
408  try {
409  Files.createDirectory(path);
410  if (UNCPathUtilities.isUNC(path)) {
411  // if the UNC path is using an IP address, convert to hostname
412  path = uncPathUtilities.ipToHostName(path);
413  if (path == null) {
414  throw new IngestModule.IngestModuleException(Bundle.PhotoRecIngestModule_nonHostnameUNCPathUsed());
415  }
416  if (false == FileUtil.hasReadWriteAccess(path)) {
417  throw new IngestModule.IngestModuleException(
418  Bundle.PhotoRecIngestModule_PermissionsNotSufficient() + SEP + path.toString() + SEP
419  + Bundle.PhotoRecIngestModule_PermissionsNotSufficientSeeReference()
420  );
421  }
422  }
423  } catch (FileAlreadyExistsException ex) {
424  // No worries.
425  } catch (IOException | SecurityException | UnsupportedOperationException ex) {
426  throw new IngestModule.IngestModuleException(Bundle.cannotCreateOutputDir_message(ex.getLocalizedMessage()), ex);
427  }
428  return path;
429  }
430 
440  public static File locateExecutable() throws IngestModule.IngestModuleException {
441  File exeFile = null;
442  Path execName = null;
443  String photorec_linux_directory = "/usr/bin";
444  if (PlatformUtil.isWindowsOS()) {
445  execName = Paths.get(PHOTOREC_DIRECTORY, PHOTOREC_EXECUTABLE);
446  exeFile = InstalledFileLocator.getDefault().locate(execName.toString(), PhotoRecCarverFileIngestModule.class.getPackage().getName(), false);
447  } else {
448  File usrBin = new File("/usr/bin/photorec");
449  File usrLocalBin = new File("/usr/local/bin/photorec");
450  if (usrBin.canExecute() && usrBin.exists() && !usrBin.isDirectory()) {
451  photorec_linux_directory = "/usr/bin";
452  }else if(usrLocalBin.canExecute() && usrLocalBin.exists() && !usrLocalBin.isDirectory()){
453  photorec_linux_directory = "/usr/local/bin";
454  }else{
455  throw new IngestModule.IngestModuleException("Photorec not found");
456  }
457  execName = Paths.get(photorec_linux_directory, PHOTOREC_LINUX_EXECUTABLE);
458  exeFile = new File(execName.toString());
459  }
460 
461  if (null == exeFile) {
462  throw new IngestModule.IngestModuleException(Bundle.missingExecutable_message());
463  }
464 
465 
466  if (!exeFile.canExecute()) {
467  throw new IngestModule.IngestModuleException(Bundle.cannotRunExecutable_message());
468  }
469 
470  return exeFile;
471  }
472 
473 }
void fireModuleContentEvent(ModuleContentEvent moduleContentEvent)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
synchronized Path ipToHostName(Path inputPath)
static synchronized IngestServices getInstance()

Copyright © 2012-2016 Basis Technology. Generated on: Tue Feb 20 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.