Autopsy  4.7.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
SevenZipExtractor.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2013-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.embeddedfileextractor;
20 
21 import java.io.File;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.io.OutputStream;
25 import java.nio.file.Files;
26 import java.nio.file.Paths;
27 import java.util.ArrayList;
28 import java.util.Collections;
29 import java.util.Date;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.concurrent.ConcurrentHashMap;
33 import java.util.logging.Level;
34 import net.sf.sevenzipjbinding.ArchiveFormat;
35 import static net.sf.sevenzipjbinding.ArchiveFormat.RAR;
36 import net.sf.sevenzipjbinding.ISequentialOutStream;
37 import net.sf.sevenzipjbinding.ISevenZipInArchive;
38 import net.sf.sevenzipjbinding.SevenZip;
39 import net.sf.sevenzipjbinding.SevenZipException;
40 import net.sf.sevenzipjbinding.SevenZipNativeInitializationException;
41 import net.sf.sevenzipjbinding.simple.ISimpleInArchive;
42 import net.sf.sevenzipjbinding.simple.ISimpleInArchiveItem;
43 import net.sf.sevenzipjbinding.ExtractOperationResult;
44 import org.netbeans.api.progress.ProgressHandle;
45 import org.openide.util.NbBundle;
46 import org.openide.util.NbBundle.Messages;
61 import org.sleuthkit.datamodel.AbstractFile;
62 import org.sleuthkit.datamodel.BlackboardArtifact;
63 import org.sleuthkit.datamodel.BlackboardAttribute;
64 import org.sleuthkit.datamodel.Content;
65 import org.sleuthkit.datamodel.DerivedFile;
66 import org.sleuthkit.datamodel.EncodedFileOutputStream;
67 import org.sleuthkit.datamodel.ReadContentInputStream;
68 import org.sleuthkit.datamodel.TskCoreException;
69 import org.sleuthkit.datamodel.TskData;
70 
71 class SevenZipExtractor {
72 
73  private static final Logger logger = Logger.getLogger(SevenZipExtractor.class.getName());
74  private IngestServices services = IngestServices.getInstance();
75  private final IngestJobContext context;
76  private final FileTypeDetector fileTypeDetector;
77  //encryption type strings
78  private static final String ENCRYPTION_FILE_LEVEL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
79  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFileLevel");
80  private static final String ENCRYPTION_FULL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
81  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFull");
82  //zip bomb detection
83  private static final int MAX_DEPTH = 4;
84  private static final int MAX_COMPRESSION_RATIO = 600;
85  private static final long MIN_COMPRESSION_RATIO_SIZE = 500 * 1000000L;
86  private static final long MIN_FREE_DISK_SPACE = 1 * 1000 * 1000000L; //1GB
87 
88  private String moduleDirRelative;
89  private String moduleDirAbsolute;
90 
91  private Blackboard blackboard;
92 
93  private String getLocalRootAbsPath(String uniqueArchiveFileName) {
94  return moduleDirAbsolute + File.separator + uniqueArchiveFileName;
95  }
96 
101 
102  ZIP("application/zip"), //NON-NLS
103  SEVENZ("application/x-7z-compressed"), //NON-NLS
104  GZIP("application/gzip"), //NON-NLS
105  XGZIP("application/x-gzip"), //NON-NLS
106  XBZIP2("application/x-bzip2"), //NON-NLS
107  XTAR("application/x-tar"), //NON-NLS
108  XGTAR("application/x-gtar"),
109  XRAR("application/x-rar-compressed"); //NON-NLS
110 
111  private final String mimeType;
112 
113  SupportedArchiveExtractionFormats(final String mimeType) {
114  this.mimeType = mimeType;
115  }
116 
117  @Override
118  public String toString() {
119  return this.mimeType;
120  }
121  // TODO Expand to support more formats after upgrading Tika
122  }
123 
124  SevenZipExtractor(IngestJobContext context, FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute) throws SevenZipNativeInitializationException {
125  if (!SevenZip.isInitializedSuccessfully() && (SevenZip.getLastInitializationException() == null)) {
126  SevenZip.initSevenZipFromPlatformJAR();
127  }
128  this.context = context;
129  this.fileTypeDetector = fileTypeDetector;
130  this.moduleDirRelative = moduleDirRelative;
131  this.moduleDirAbsolute = moduleDirAbsolute;
132  }
133 
142  boolean isSevenZipExtractionSupported(AbstractFile file) {
143  String fileMimeType = fileTypeDetector.getMIMEType(file);
144  for (SupportedArchiveExtractionFormats mimeType : SupportedArchiveExtractionFormats.values()) {
145  if (mimeType.toString().equals(fileMimeType)) {
146  return true;
147  }
148  }
149  return false;
150  }
151 
164  private boolean isZipBombArchiveItemCheck(AbstractFile archiveFile, ISimpleInArchiveItem archiveFileItem) {
165  try {
166  final Long archiveItemSize = archiveFileItem.getSize();
167 
168  //skip the check for small files
169  if (archiveItemSize == null || archiveItemSize < MIN_COMPRESSION_RATIO_SIZE) {
170  return false;
171  }
172 
173  final Long archiveItemPackedSize = archiveFileItem.getPackedSize();
174 
175  if (archiveItemPackedSize == null || archiveItemPackedSize <= 0) {
176  logger.log(Level.WARNING, "Cannot getting compression ratio, cannot detect if zipbomb: {0}, item: {1}", new Object[]{archiveFile.getName(), archiveFileItem.getPath()}); //NON-NLS
177  return false;
178  }
179 
180  int cRatio = (int) (archiveItemSize / archiveItemPackedSize);
181 
182  if (cRatio >= MAX_COMPRESSION_RATIO) {
183  String itemName = archiveFileItem.getPath();
184  logger.log(Level.INFO, "Possible zip bomb detected, compression ration: {0} for in archive item: {1}", new Object[]{cRatio, itemName}); //NON-NLS
185  String msg = NbBundle.getMessage(SevenZipExtractor.class,
186  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg", archiveFile.getName(), itemName);
187  String path;
188  try {
189  path = archiveFile.getUniquePath();
190  } catch (TskCoreException ex) {
191  path = archiveFile.getParentPath() + archiveFile.getName();
192  }
193  String details = NbBundle.getMessage(SevenZipExtractor.class,
194  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails", cRatio, path);
195  //MessageNotifyUtil.Notify.error(msg, details);
196  services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
197  return true;
198  } else {
199  return false;
200  }
201 
202  } catch (SevenZipException ex) {
203  logger.log(Level.WARNING, "Error getting archive item size and cannot detect if zipbomb. ", ex); //NON-NLS
204  return false;
205  }
206  }
207 
216  private ArchiveFormat get7ZipOptions(AbstractFile archiveFile) {
217  // try to get the file type from the BB
218  String detectedFormat;
219  detectedFormat = archiveFile.getMIMEType();
220 
221  if (detectedFormat == null) {
222  logger.log(Level.WARNING, "Could not detect format for file: {0}", archiveFile); //NON-NLS
223 
224  // if we don't have attribute info then use file extension
225  String extension = archiveFile.getNameExtension();
226  if ("rar".equals(extension)) //NON-NLS
227  {
228  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
229  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality
230  return RAR;
231  }
232 
233  // Otherwise open the archive using 7zip's built-in auto-detect functionality
234  return null;
235  } else if (detectedFormat.contains("application/x-rar-compressed")) //NON-NLS
236  {
237  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
238  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality
239  return RAR;
240  }
241 
242  // Otherwise open the archive using 7zip's built-in auto-detect functionality
243  return null;
244  }
245 
256  private long getRootArchiveId(AbstractFile file) throws TskCoreException {
257  long id = file.getId();
258  Content parentContent = file.getParent();
259  while (parentContent != null) {
260  id = parentContent.getId();
261  parentContent = parentContent.getParent();
262  }
263  return id;
264  }
265 
280  private List<AbstractFile> getAlreadyExtractedFiles(AbstractFile archiveFile, String archiveFilePath) throws TskCoreException, NoCurrentCaseException {
281  //check if already has derived files, skip
282  //check if local unpacked dir exists
283  if (archiveFile.hasChildren() && new File(moduleDirAbsolute, EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)).exists()) {
284  return Case.getCurrentCaseThrows().getServices().getFileManager().findFilesByParentPath(getRootArchiveId(archiveFile), archiveFilePath);
285  }
286  return new ArrayList<>();
287  }
288 
296  private String getArchiveFilePath(AbstractFile archiveFile) {
297  try {
298  return archiveFile.getUniquePath();
299  } catch (TskCoreException ex) {
300  return archiveFile.getParentPath() + archiveFile.getName();
301  }
302  }
303 
310  private void makeLocalDirectories(String uniqueArchiveFileName) {
311  final String localRootAbsPath = getLocalRootAbsPath(uniqueArchiveFileName);
312  final File localRoot = new File(localRootAbsPath);
313  if (!localRoot.exists()) {
314  localRoot.mkdirs();
315  }
316  }
317 
330  private String getPathInArchive(ISimpleInArchiveItem item, int itemNumber, AbstractFile archiveFile) throws SevenZipException {
331  String pathInArchive = item.getPath();
332 
333  if (pathInArchive == null || pathInArchive.isEmpty()) {
334  //some formats (.tar.gz) may not be handled correctly -- file in archive has no name/path
335  //handle this for .tar.gz and tgz but assuming the child is tar,
336  //otherwise, unpack using itemNumber as name
337 
338  //TODO this should really be signature based, not extension based
339  String archName = archiveFile.getName();
340  int dotI = archName.lastIndexOf(".");
341  String useName = null;
342  if (dotI != -1) {
343  String base = archName.substring(0, dotI);
344  String ext = archName.substring(dotI);
345  int colonIndex = ext.lastIndexOf(":");
346  if (colonIndex != -1) {
347  // If alternate data stream is found, fix the name
348  // so Windows doesn't choke on the colon character.
349  ext = ext.substring(0, colonIndex);
350  }
351  switch (ext) {
352  case ".gz": //NON-NLS
353  useName = base;
354  break;
355  case ".tgz": //NON-NLS
356  useName = base + ".tar"; //NON-NLS
357  break;
358  case ".bz2": //NON-NLS
359  useName = base;
360  break;
361  }
362  }
363  if (useName == null) {
364  pathInArchive = "/" + archName + "/" + Integer.toString(itemNumber);
365  } else {
366  pathInArchive = "/" + useName;
367  }
368  String msg = NbBundle.getMessage(SevenZipExtractor.class,
369  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.unknownPath.msg",
370  getArchiveFilePath(archiveFile), pathInArchive);
371  logger.log(Level.WARNING, msg);
372  }
373  return pathInArchive;
374  }
375 
376  /*
377  * Get the String that will represent the key for the hashmap which keeps
378  * track of existing files from an AbstractFile
379  */
380  private String getKeyAbstractFile(AbstractFile fileInDatabase) {
381  return fileInDatabase == null ? null : fileInDatabase.getParentPath() + fileInDatabase.getName();
382  }
383 
384  /*
385  * Get the String that will represent the key for the hashmap which keeps
386  * track of existing files from an unpacked node and the archiveFilePath
387  */
388  private String getKeyFromUnpackedNode(UnpackedTree.UnpackedNode node, String archiveFilePath) {
389  return node == null ? null : archiveFilePath + "/" + node.getFileName();
390  }
391 
406  private SevenZipExtractor.UnpackedTree.UnpackedNode unpackNode(ISimpleInArchiveItem item, SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode, String password, long freeDiskSpace, String uniqueExtractedName) throws SevenZipException {
407  //unpack locally if a file
408  final String localAbsPath = moduleDirAbsolute + File.separator + uniqueExtractedName;
409  final String localRelPath = moduleDirRelative + File.separator + uniqueExtractedName;
410  final Date createTime = item.getCreationTime();
411  final Date accessTime = item.getLastAccessTime();
412  final Date writeTime = item.getLastWriteTime();
413  final long createtime = createTime == null ? 0L : createTime.getTime() / 1000;
414  final long modtime = writeTime == null ? 0L : writeTime.getTime() / 1000;
415  final long accesstime = accessTime == null ? 0L : accessTime.getTime() / 1000;
416  SevenZipExtractor.UnpackStream unpackStream = null;
417  boolean isDir = item.isFolder();
418  if (!isDir) {
419  try {
420  // NOTE: item.getSize() may return null in case of certain
421  // archiving formats. Eg: BZ2
422  if (item.getSize() != null) {
423  unpackStream = new SevenZipExtractor.KnownSizeUnpackStream(localAbsPath, item.getSize());
424  } else {
425  unpackStream = new SevenZipExtractor.UnknownSizeUnpackStream(localAbsPath, freeDiskSpace);
426  }
427  ExtractOperationResult result;
428  if (password == null) {
429  result = item.extractSlow(unpackStream);
430  } else {
431  result = item.extractSlow(unpackStream, password);
432  }
433  if (result != ExtractOperationResult.OK) {
434  logger.log(Level.WARNING, "Extraction of : {0} encountered error {1}", new Object[]{localAbsPath, result}); //NON-NLS
435  return null;
436  }
437 
438  } catch (SevenZipException e) {
439  //could be something unexpected with this file, move on
440  logger.log(Level.WARNING, "Could not extract file from archive: " + localAbsPath, e); //NON-NLS
441  } finally {
442  if (unpackStream != null) {
443  //record derived data in unode, to be traversed later after unpacking the archive
444  unpackedNode.addDerivedInfo(unpackStream.getSize(), !isDir,
445  0L, createtime, accesstime, modtime, localRelPath);
446  unpackStream.close();
447  }
448  }
449  } else { // this is a directory, size is always 0
450  unpackedNode.addDerivedInfo(0, !isDir, 0L, createtime, accesstime, modtime, localRelPath);
451  }
452  return unpackedNode;
453  }
454 
464  void unpack(AbstractFile archiveFile, ConcurrentHashMap<Long, Archive> depthMap) {
465  unpack(archiveFile, depthMap, null);
466  }
467 
479  @Messages({"SevenZipExtractor.indexError.message=Failed to index encryption detected artifact for keyword search."})
480  boolean unpack(AbstractFile archiveFile, ConcurrentHashMap<Long, Archive> depthMap, String password) {
481  boolean unpackSuccessful = true; //initialized to true change to false if any files fail to extract and
482  boolean hasEncrypted = false;
483  boolean fullEncryption = true;
484  boolean progressStarted = false;
485  int processedItems = 0;
486  final String archiveFilePath = getArchiveFilePath(archiveFile);
487  final String escapedArchiveFilePath = FileUtil.escapeFileName(archiveFilePath);
488  HashMap<String, ZipFileStatusWrapper> statusMap = new HashMap<>();
489  List<AbstractFile> unpackedFiles = Collections.<AbstractFile>emptyList();
490  ISevenZipInArchive inArchive = null;
491 
492  SevenZipContentReadStream stream = null;
493  final ProgressHandle progress = ProgressHandle.createHandle(Bundle.EmbeddedFileExtractorIngestModule_ArchiveExtractor_moduleName());
494  //recursion depth check for zip bomb
495  final long archiveId = archiveFile.getId();
496  Archive parentAr;
497  try {
498  blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard();
499  } catch (NoCurrentCaseException ex) {
500  logger.log(Level.INFO, "Exception while getting open case.", ex); //NON-NLS
501  unpackSuccessful = false;
502  return unpackSuccessful;
503  }
504  try {
505 
506  List<AbstractFile> existingFiles = getAlreadyExtractedFiles(archiveFile, archiveFilePath);
507  for (AbstractFile file : existingFiles) {
508  statusMap.put(getKeyAbstractFile(file), new ZipFileStatusWrapper(file, ZipFileStatus.EXISTS));
509  }
510  } catch (TskCoreException e) {
511  logger.log(Level.INFO, "Error checking if file already has been processed, skipping: {0}", escapedArchiveFilePath); //NON-NLS
512  unpackSuccessful = false;
513  return unpackSuccessful;
514  } catch (NoCurrentCaseException ex) {
515  logger.log(Level.INFO, "No open case was found while trying to unpack the archive file {0}", escapedArchiveFilePath); //NON-NLS
516  unpackSuccessful = false;
517  return unpackSuccessful;
518  }
519  parentAr = depthMap.get(archiveId);
520  if (parentAr == null) {
521  parentAr = new Archive(archiveId, 0);
522  depthMap.put(archiveId, parentAr);
523  } else if (parentAr.getDepth() == MAX_DEPTH) {
524  String msg = NbBundle.getMessage(SevenZipExtractor.class,
525  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnMsg.zipBomb", archiveFile.getName());
526  String details = NbBundle.getMessage(SevenZipExtractor.class,
527  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnDetails.zipBomb",
528  parentAr.getDepth(), escapedArchiveFilePath);
529  //MessageNotifyUtil.Notify.error(msg, details);
530  services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
531  unpackSuccessful = false;
532  return unpackSuccessful;
533  }
534  try {
535  stream = new SevenZipContentReadStream(new ReadContentInputStream(archiveFile));
536  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
537  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality.
538  // All other archive formats are still opened using 7zip built-in auto-detect functionality.
539  ArchiveFormat options = get7ZipOptions(archiveFile);
540  if (password == null) {
541  inArchive = SevenZip.openInArchive(options, stream);
542  } else {
543  inArchive = SevenZip.openInArchive(options, stream, password);
544  }
545  int numItems = inArchive.getNumberOfItems();
546  logger.log(Level.INFO, "Count of items in archive: {0}: {1}", new Object[]{escapedArchiveFilePath, numItems}); //NON-NLS
547  progress.start(numItems);
548  progressStarted = true;
549  final ISimpleInArchive simpleInArchive = inArchive.getSimpleInterface();
550 
551  //setup the archive local root folder
552  final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
553  try {
554  makeLocalDirectories(uniqueArchiveFileName);
555  } catch (SecurityException e) {
556  logger.log(Level.SEVERE, "Error setting up output path for archive root: {0}", getLocalRootAbsPath(uniqueArchiveFileName)); //NON-NLS
557  //bail
558  unpackSuccessful = false;
559  return unpackSuccessful;
560  }
561 
562  //initialize tree hierarchy to keep track of unpacked file structure
563  SevenZipExtractor.UnpackedTree unpackedTree = new SevenZipExtractor.UnpackedTree(moduleDirRelative + "/" + uniqueArchiveFileName, archiveFile);
564 
565  long freeDiskSpace;
566  try {
567  freeDiskSpace = services.getFreeDiskSpace();
568  } catch (NullPointerException ex) {
569  //If ingest has not been run at least once getFreeDiskSpace() will throw a null pointer exception
570  //currently getFreeDiskSpace always returns DISK_FREE_SPACE_UNKNOWN
571  freeDiskSpace = IngestMonitor.DISK_FREE_SPACE_UNKNOWN;
572  }
573  //unpack and process every item in archive
574  int itemNumber = 0;
575 
576  for (ISimpleInArchiveItem item : simpleInArchive.getArchiveItems()) {
577  String pathInArchive = getPathInArchive(item, itemNumber, archiveFile);
578 
579  //query for path in db
580  ++itemNumber;
581 
582  //check if possible zip bomb
583  if (isZipBombArchiveItemCheck(archiveFile, item)) {
584  unpackSuccessful = false;
585  return unpackSuccessful;
586  }
587  SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode = unpackedTree.addNode(pathInArchive);
588  //update progress bar
589  progress.progress(archiveFile.getName() + ": " + item.getPath(), processedItems);
590 
591  final boolean isEncrypted = item.isEncrypted();
592 
593  if (isEncrypted && password == null) {
594  logger.log(Level.WARNING, "Skipping encrypted file in archive: {0}", pathInArchive); //NON-NLS
595  hasEncrypted = true;
596  unpackSuccessful = false;
597  continue;
598  } else {
599  fullEncryption = false;
600  }
601  // NOTE: item.getSize() may return null in case of certain
602  // archiving formats. Eg: BZ2
603  //check if unpacking this file will result in out of disk space
604  //this is additional to zip bomb prevention mechanism
605  if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && item.getSize() != null && item.getSize() > 0) { //if free space is known and file is not empty.
606  long newDiskSpace = freeDiskSpace - item.getSize();
607  if (newDiskSpace < MIN_FREE_DISK_SPACE) {
608  String msg = NbBundle.getMessage(SevenZipExtractor.class,
609  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.msg",
610  escapedArchiveFilePath, item.getPath());
611  String details = NbBundle.getMessage(SevenZipExtractor.class,
612  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.details");
613  //MessageNotifyUtil.Notify.error(msg, details);
614  services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
615  logger.log(Level.INFO, "Skipping archive item due to insufficient disk space: {0}, {1}", new String[]{escapedArchiveFilePath, item.getPath()}); //NON-NLS
616  logger.log(Level.INFO, "Available disk space: {0}", new Object[]{freeDiskSpace}); //NON-NLS
617  unpackSuccessful = false;
618  continue; //skip this file
619  } else {
620  //update est. disk space during this archive, so we don't need to poll for every file extracted
621  freeDiskSpace = newDiskSpace;
622  }
623  }
624  final String uniqueExtractedName = FileUtil.escapeFileName(uniqueArchiveFileName + File.separator + (item.getItemIndex() / 1000) + File.separator + item.getItemIndex() + "_" + new File(pathInArchive).getName());
625 
626  //create local dirs and empty files before extracted
627  File localFile = new java.io.File(moduleDirAbsolute + File.separator + uniqueExtractedName);
628  //cannot rely on files in top-bottom order
629  if (!localFile.exists()) {
630  try {
631  if (item.isFolder()) {
632  localFile.mkdirs();
633  } else {
634  localFile.getParentFile().mkdirs();
635  try {
636  localFile.createNewFile();
637  } catch (IOException e) {
638  logger.log(Level.SEVERE, "Error creating extracted file: " + localFile.getAbsolutePath(), e); //NON-NLS
639  }
640  }
641  } catch (SecurityException e) {
642  logger.log(Level.SEVERE, "Error setting up output path for unpacked file: {0}", pathInArchive); //NON-NLS
643  //TODO consider bail out / msg to the user
644  }
645  }
646  // skip the rest of this loop if we couldn't create the file
647  if (localFile.exists() == false) {
648  continue;
649  }
650  //find this node in the hierarchy, create if neede;
651  unpackedNode = unpackNode(item, unpackedNode, password,
652  freeDiskSpace, uniqueExtractedName);
653  if (unpackedNode == null) {
654  unpackSuccessful = false;
655  }
656 
657  //update units for progress bar
658  ++processedItems;
659  }
660  // add them to the DB. We wait until the end so that we have the metadata on all of the
661  // intermediate nodes since the order is not guaranteed
662  try {
663  unpackedTree.updateOrAddFileToCaseRec(statusMap, archiveFilePath);
664  unpackedFiles = unpackedTree.getAllFileObjects();
665  //check if children are archives, update archive depth tracking
666  for (AbstractFile unpackedFile : unpackedFiles) {
667  if (unpackedFile == null) {
668  continue;
669  }
670  if (isSevenZipExtractionSupported(unpackedFile)) {
671  Archive child = new Archive(unpackedFile.getId(), parentAr.getDepth() + 1);
672  parentAr.addChild(child);
673  depthMap.put(unpackedFile.getId(), child);
674  }
675  }
676 
677  } catch (TskCoreException | NoCurrentCaseException e) {
678  logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", e); //NON-NLS
679  //TODO decide if anything to cleanup, for now bailing
680  }
681 
682  } catch (SevenZipException ex) {
683  logger.log(Level.WARNING, "Error unpacking file: " + archiveFile, ex); //NON-NLS
684  //inbox message
685 
686  // print a message if the file is allocated
687  if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
688  String msg = NbBundle.getMessage(SevenZipExtractor.class,
689  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
690  archiveFile.getName());
691  String details = NbBundle.getMessage(SevenZipExtractor.class,
692  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.details",
693  escapedArchiveFilePath, ex.getMessage());
694  services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
695  }
696  } finally {
697  if (inArchive != null) {
698  try {
699  inArchive.close();
700  } catch (SevenZipException e) {
701  logger.log(Level.SEVERE, "Error closing archive: " + archiveFile, e); //NON-NLS
702  }
703  }
704 
705  if (stream != null) {
706  try {
707  stream.close();
708  } catch (IOException ex) {
709  logger.log(Level.SEVERE, "Error closing stream after unpacking archive: " + archiveFile, ex); //NON-NLS
710  }
711  }
712 
713  //close progress bar
714  if (progressStarted) {
715  progress.finish();
716  }
717  }
718 
719  //create artifact and send user message
720  if (hasEncrypted) {
721  String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
722  try {
723  BlackboardArtifact artifact = archiveFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
724  artifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, EmbeddedFileExtractorModuleFactory.getModuleName(), encryptionType));
725 
726  try {
727  // index the artifact for keyword search
728  blackboard.indexArtifact(artifact);
729  } catch (Blackboard.BlackboardException ex) {
730  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
731  MessageNotifyUtil.Notify.error(
732  Bundle.SevenZipExtractor_indexError_message(), artifact.getDisplayName());
733  }
734 
735  services.fireModuleDataEvent(new ModuleDataEvent(EmbeddedFileExtractorModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
736  } catch (TskCoreException ex) {
737  logger.log(Level.SEVERE, "Error creating blackboard artifact for encryption detected for file: " + escapedArchiveFilePath, ex); //NON-NLS
738  }
739 
740  String msg = NbBundle.getMessage(SevenZipExtractor.class,
741  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
742  String details = NbBundle.getMessage(SevenZipExtractor.class,
743  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.details",
744  archiveFile.getName(), EmbeddedFileExtractorModuleFactory.getModuleName());
745  services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
746  }
747 
748  // adding unpacked extracted derived files to the job after closing relevant resources.
749  if (!unpackedFiles.isEmpty()) {
750  //currently sending a single event for all new files
751  services.fireModuleContentEvent(new ModuleContentEvent(archiveFile));
752  if (context != null) {
753  context.addFilesToJob(unpackedFiles);
754  }
755  }
756  return unpackSuccessful;
757  }
758 
762  private abstract static class UnpackStream implements ISequentialOutStream {
763 
764  private OutputStream output;
765  private String localAbsPath;
766 
767  UnpackStream(String localAbsPath) {
768  this.localAbsPath = localAbsPath;
769  try {
770  output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1);
771  } catch (IOException ex) {
772  logger.log(Level.SEVERE, "Error writing extracted file: " + localAbsPath, ex); //NON-NLS
773  }
774 
775  }
776 
777  public abstract long getSize();
778 
779  OutputStream getOutput() {
780  return output;
781  }
782 
783  String getLocalAbsPath() {
784  return localAbsPath;
785  }
786 
787  public void close() {
788  if (output != null) {
789  try {
790  output.flush();
791  output.close();
792  } catch (IOException e) {
793  logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS
794  }
795  }
796  }
797  }
798 
802  private static class UnknownSizeUnpackStream extends UnpackStream {
803 
804  private long freeDiskSpace;
805  private boolean outOfSpace = false;
806  private long bytesWritten = 0;
807 
808  UnknownSizeUnpackStream(String localAbsPath, long freeDiskSpace) {
809  super(localAbsPath);
810  this.freeDiskSpace = freeDiskSpace;
811  }
812 
813  @Override
814  public long getSize() {
815  return this.bytesWritten;
816  }
817 
818  @Override
819  public int write(byte[] bytes) throws SevenZipException {
820  try {
821  // If the content size is unknown, cautiously write to disk.
822  // Write only if byte array is less than 80% of the current
823  // free disk space.
824  if (freeDiskSpace == IngestMonitor.DISK_FREE_SPACE_UNKNOWN || bytes.length < 0.8 * freeDiskSpace) {
825  getOutput().write(bytes);
826  // NOTE: this method is called multiple times for a
827  // single extractSlow() call. Update bytesWritten and
828  // freeDiskSpace after every write operation.
829  this.bytesWritten += bytes.length;
830  this.freeDiskSpace -= bytes.length;
831  } else {
832  this.outOfSpace = true;
833  logger.log(Level.INFO, NbBundle.getMessage(
834  SevenZipExtractor.class,
835  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
836  throw new SevenZipException(
837  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
838  }
839  } catch (IOException ex) {
840  throw new SevenZipException(
841  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
842  getLocalAbsPath()), ex);
843  }
844  return bytes.length;
845  }
846 
847  @Override
848  public void close() {
849  if (getOutput() != null) {
850  try {
851  getOutput().flush();
852  getOutput().close();
853  if (this.outOfSpace) {
854  Files.delete(Paths.get(getLocalAbsPath()));
855  }
856  } catch (IOException e) {
857  logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", getLocalAbsPath()); //NON-NLS
858  }
859  }
860  }
861  }
862 
866  private static class KnownSizeUnpackStream extends UnpackStream {
867 
868  private long size;
869 
870  KnownSizeUnpackStream(String localAbsPath, long size) {
871  super(localAbsPath);
872  this.size = size;
873  }
874 
875  @Override
876  public long getSize() {
877  return this.size;
878  }
879 
880  @Override
881  public int write(byte[] bytes) throws SevenZipException {
882  try {
883  getOutput().write(bytes);
884  } catch (IOException ex) {
885  throw new SevenZipException(
886  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
887  getLocalAbsPath()), ex);
888  }
889  return bytes.length;
890  }
891  }
892 
900  private class UnpackedTree {
901 
902  final UnpackedNode rootNode;
903 
911  UnpackedTree(String localPathRoot, AbstractFile archiveFile) {
912  this.rootNode = new UnpackedNode();
913  this.rootNode.setFile(archiveFile);
914  this.rootNode.setFileName(archiveFile.getName());
915  this.rootNode.setLocalRelPath(localPathRoot);
916  }
917 
927  UnpackedNode addNode(String filePath) {
928  String[] toks = filePath.split("[\\/\\\\]");
929  List<String> tokens = new ArrayList<>();
930  for (int i = 0; i < toks.length; ++i) {
931  if (!toks[i].isEmpty()) {
932  tokens.add(toks[i]);
933  }
934  }
935  return addNode(rootNode, tokens);
936  }
937 
946  private UnpackedNode addNode(UnpackedNode parent, List<String> tokenPath) {
947  // we found all of the tokens
948  if (tokenPath.isEmpty()) {
949  return parent;
950  }
951 
952  // get the next name in the path and look it up
953  String childName = tokenPath.remove(0);
954  UnpackedNode child = parent.getChild(childName);
955  // create new node
956  if (child == null) {
957  child = new UnpackedNode(childName, parent);
958  parent.addChild(child);
959  }
960 
961  // go down one more level
962  return addNode(child, tokenPath);
963  }
964 
971  List<AbstractFile> getRootFileObjects() {
972  List<AbstractFile> ret = new ArrayList<>();
973  for (UnpackedNode child : rootNode.getChildren()) {
974  ret.add(child.getFile());
975  }
976  return ret;
977  }
978 
985  List<AbstractFile> getAllFileObjects() {
986  List<AbstractFile> ret = new ArrayList<>();
987  for (UnpackedNode child : rootNode.getChildren()) {
988  getAllFileObjectsRec(ret, child);
989  }
990  return ret;
991  }
992 
993  private void getAllFileObjectsRec(List<AbstractFile> list, UnpackedNode parent) {
994  list.add(parent.getFile());
995  for (UnpackedNode child : parent.getChildren()) {
996  getAllFileObjectsRec(list, child);
997  }
998  }
999 
1004  void updateOrAddFileToCaseRec(HashMap<String, ZipFileStatusWrapper> statusMap, String archiveFilePath) throws TskCoreException, NoCurrentCaseException {
1006  for (UnpackedNode child : rootNode.getChildren()) {
1007  updateOrAddFileToCaseRec(child, fileManager, statusMap, archiveFilePath);
1008  }
1009  }
1010 
1025  private void updateOrAddFileToCaseRec(UnpackedNode node, FileManager fileManager, HashMap<String, ZipFileStatusWrapper> statusMap, String archiveFilePath) throws TskCoreException {
1026  DerivedFile df;
1027  try {
1028  String nameInDatabase = getKeyFromUnpackedNode(node, archiveFilePath);
1029  ZipFileStatusWrapper existingFile = nameInDatabase == null ? null : statusMap.get(nameInDatabase);
1030  if (existingFile == null) {
1031  df = fileManager.addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
1032  node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
1033  node.isIsFile(), node.getParent().getFile(), "", EmbeddedFileExtractorModuleFactory.getModuleName(),
1034  "", "", TskData.EncodingType.XOR1);
1035  statusMap.put(getKeyAbstractFile(df), new ZipFileStatusWrapper(df, ZipFileStatus.EXISTS));
1036  } else {
1037  String key = getKeyAbstractFile(existingFile.getFile());
1038  if (existingFile.getStatus() == ZipFileStatus.EXISTS && existingFile.getFile().getSize() < node.getSize()) {
1039  existingFile.setStatus(ZipFileStatus.UPDATE);
1040  statusMap.put(key, existingFile);
1041  }
1042  if (existingFile.getStatus() == ZipFileStatus.UPDATE) {
1043  //if the we are updating a file and its mime type was octet-stream we want to re-type it
1044  String mimeType = existingFile.getFile().getMIMEType().equalsIgnoreCase("application/octet-stream") ? null : existingFile.getFile().getMIMEType();
1045  df = fileManager.updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
1046  node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
1047  node.isIsFile(), mimeType, "", EmbeddedFileExtractorModuleFactory.getModuleName(),
1048  "", "", TskData.EncodingType.XOR1);
1049  } else {
1050  //ALREADY CURRENT - SKIP
1051  statusMap.put(key, new ZipFileStatusWrapper(existingFile.getFile(), ZipFileStatus.SKIP));
1052  df = (DerivedFile) existingFile.getFile();
1053  }
1054  }
1055  node.setFile(df);
1056  } catch (TskCoreException ex) {
1057  logger.log(Level.SEVERE, "Error adding a derived file to db:" + node.getFileName(), ex); //NON-NLS
1058  throw new TskCoreException(
1059  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
1060  node.getFileName()), ex);
1061  }
1062  //recurse adding the children if this file was incomplete the children presumably need to be added
1063  for (UnpackedNode child : node.getChildren()) {
1064  updateOrAddFileToCaseRec(child, fileManager, statusMap, getKeyFromUnpackedNode(node, archiveFilePath));
1065  }
1066  }
1067 
1071  private class UnpackedNode {
1072 
1073  private String fileName;
1074  private AbstractFile file;
1075  private final List<UnpackedNode> children = new ArrayList<>();
1076  private String localRelPath = "";
1077  private long size;
1078  private long ctime, crtime, atime, mtime;
1079  private boolean isFile;
1081 
1082  //root constructor
1083  UnpackedNode() {
1084  }
1085 
1086  //child node constructor
1087  UnpackedNode(String fileName, UnpackedNode parent) {
1088  this.fileName = fileName;
1089  this.parent = parent;
1090  this.localRelPath = parent.getLocalRelPath() + File.separator + fileName;
1091  }
1092 
1093  long getCtime() {
1094  return ctime;
1095  }
1096 
1097  long getCrtime() {
1098  return crtime;
1099  }
1100 
1101  long getAtime() {
1102  return atime;
1103  }
1104 
1105  long getMtime() {
1106  return mtime;
1107  }
1108 
1109  void setFileName(String fileName) {
1110  this.fileName = fileName;
1111  }
1112 
1118  void addChild(UnpackedNode child) {
1119  children.add(child);
1120  }
1121 
1128  List<UnpackedNode> getChildren() {
1129  return children;
1130  }
1131 
1137  UnpackedNode getParent() {
1138  return parent;
1139  }
1140 
1141  void addDerivedInfo(long size,
1142  boolean isFile,
1143  long ctime, long crtime, long atime, long mtime, String relLocalPath) {
1144  this.size = size;
1145  this.isFile = isFile;
1146  this.ctime = ctime;
1147  this.crtime = crtime;
1148  this.atime = atime;
1149  this.mtime = mtime;
1150  this.localRelPath = relLocalPath;
1151  }
1152 
1153  void setFile(AbstractFile file) {
1154  this.file = file;
1155  }
1156 
1164  UnpackedNode getChild(String childFileName) {
1165  UnpackedNode ret = null;
1166  for (UnpackedNode child : children) {
1167  if (child.getFileName().equals(childFileName)) {
1168  ret = child;
1169  break;
1170  }
1171  }
1172  return ret;
1173  }
1174 
1175  String getFileName() {
1176  return fileName;
1177  }
1178 
1179  AbstractFile getFile() {
1180  return file;
1181  }
1182 
1183  String getLocalRelPath() {
1184  return localRelPath;
1185  }
1186 
1193  void setLocalRelPath(String localRelativePath) {
1194  localRelPath = localRelativePath;
1195  }
1196 
1197  long getSize() {
1198  return size;
1199  }
1200 
1201  boolean isIsFile() {
1202  return isFile;
1203  }
1204  }
1205  }
1206 
1211  static class Archive {
1212 
1213  //depth will be 0 for the root archive unpack was called on, and increase as unpack recurses down through archives contained within
1214  private final int depth;
1215  private final long objectId;
1216  private final List<Archive> children;
1217 
1227  Archive(long objectId, int depth) {
1228  this.objectId = objectId;
1229  this.children = new ArrayList<>();
1230  this.depth = depth;
1231  }
1232 
1239  void addChild(Archive child) {
1240  children.add(child);
1241  }
1242 
1248  long getObjectId() {
1249  return objectId;
1250  }
1251 
1259  int getDepth() {
1260  return depth;
1261  }
1262  }
1263 
1268  private final class ZipFileStatusWrapper {
1269 
1270  private final AbstractFile abstractFile;
1272 
1280  private ZipFileStatusWrapper(AbstractFile file, ZipFileStatus status) {
1281  abstractFile = file;
1282  zipStatus = status;
1283  }
1284 
1290  private AbstractFile getFile() {
1291  return abstractFile;
1292  }
1293 
1300  return zipStatus;
1301  }
1302 
1308  private void setStatus(ZipFileStatus status) {
1309  zipStatus = status;
1310  }
1311 
1312  }
1313 
1318  private enum ZipFileStatus {
1319  UPDATE, //Should be updated //NON-NLS
1320  SKIP, //File is current can be skipped //NON-NLS
1321  EXISTS //File exists but it is unknown if it is current //NON-NLS
1322  }
1323 }
void updateOrAddFileToCaseRec(UnpackedNode node, FileManager fileManager, HashMap< String, ZipFileStatusWrapper > statusMap, String archiveFilePath)
synchronized DerivedFile addDerivedFile(String fileName, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, Content parentObj, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
UnpackedNode addNode(UnpackedNode parent, List< String > tokenPath)
synchronized DerivedFile updateDerivedFile(DerivedFile derivedFile, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, String mimeType, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)

Copyright © 2012-2016 Basis Technology. Generated on: Mon Jun 18 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.