Autopsy  4.6.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
SevenZipExtractor.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2013-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.embeddedfileextractor;
20 
21 import java.io.File;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.io.OutputStream;
25 import java.nio.file.Files;
26 import java.nio.file.Paths;
27 import java.util.ArrayList;
28 import java.util.Collections;
29 import java.util.Date;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.logging.Level;
33 import net.sf.sevenzipjbinding.ArchiveFormat;
34 import static net.sf.sevenzipjbinding.ArchiveFormat.RAR;
35 import net.sf.sevenzipjbinding.ISequentialOutStream;
36 import net.sf.sevenzipjbinding.ISevenZipInArchive;
37 import net.sf.sevenzipjbinding.SevenZip;
38 import net.sf.sevenzipjbinding.SevenZipException;
39 import net.sf.sevenzipjbinding.SevenZipNativeInitializationException;
40 import net.sf.sevenzipjbinding.simple.ISimpleInArchive;
41 import net.sf.sevenzipjbinding.simple.ISimpleInArchiveItem;
42 import net.sf.sevenzipjbinding.ExtractOperationResult;
43 import org.netbeans.api.progress.ProgressHandle;
44 import org.openide.util.NbBundle;
45 import org.openide.util.NbBundle.Messages;
60 import org.sleuthkit.datamodel.AbstractFile;
61 import org.sleuthkit.datamodel.BlackboardArtifact;
62 import org.sleuthkit.datamodel.BlackboardAttribute;
63 import org.sleuthkit.datamodel.Content;
64 import org.sleuthkit.datamodel.DerivedFile;
65 import org.sleuthkit.datamodel.EncodedFileOutputStream;
66 import org.sleuthkit.datamodel.ReadContentInputStream;
67 import org.sleuthkit.datamodel.TskCoreException;
68 import org.sleuthkit.datamodel.TskData;
69 
70 class SevenZipExtractor {
71 
72  private static final Logger logger = Logger.getLogger(SevenZipExtractor.class.getName());
73  private IngestServices services = IngestServices.getInstance();
74  private final IngestJobContext context;
75  private final FileTypeDetector fileTypeDetector;
76  static final String[] SUPPORTED_EXTENSIONS = {"zip", "rar", "arj", "7z", "7zip", "gzip", "gz", "bzip2", "tar", "tgz",}; // NON-NLS
77  //encryption type strings
78  private static final String ENCRYPTION_FILE_LEVEL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
79  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFileLevel");
80  private static final String ENCRYPTION_FULL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
81  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFull");
82  //zip bomb detection
83  private static final int MAX_DEPTH = 4;
84  private static final int MAX_COMPRESSION_RATIO = 600;
85  private static final long MIN_COMPRESSION_RATIO_SIZE = 500 * 1000000L;
86  private static final long MIN_FREE_DISK_SPACE = 1 * 1000 * 1000000L; //1GB
87  //counts archive depth
88  private ArchiveDepthCountTree archiveDepthCountTree;
89 
90  private String moduleDirRelative;
91  private String moduleDirAbsolute;
92 
93  private Blackboard blackboard;
94 
95  private String getLocalRootAbsPath(String uniqueArchiveFileName) {
96  return moduleDirAbsolute + File.separator + uniqueArchiveFileName;
97  }
98 
103 
104  ZIP("application/zip"), //NON-NLS
105  SEVENZ("application/x-7z-compressed"), //NON-NLS
106  GZIP("application/gzip"), //NON-NLS
107  XGZIP("application/x-gzip"), //NON-NLS
108  XBZIP2("application/x-bzip2"), //NON-NLS
109  XTAR("application/x-tar"), //NON-NLS
110  XGTAR("application/x-gtar"),
111  XRAR("application/x-rar-compressed"); //NON-NLS
112 
113  private final String mimeType;
114 
115  SupportedArchiveExtractionFormats(final String mimeType) {
116  this.mimeType = mimeType;
117  }
118 
119  @Override
120  public String toString() {
121  return this.mimeType;
122  }
123  // TODO Expand to support more formats after upgrading Tika
124  }
125 
126  SevenZipExtractor(IngestJobContext context, FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute) throws SevenZipNativeInitializationException {
127  if (!SevenZip.isInitializedSuccessfully() && (SevenZip.getLastInitializationException() == null)) {
128  SevenZip.initSevenZipFromPlatformJAR();
129  }
130  this.context = context;
131  this.fileTypeDetector = fileTypeDetector;
132  this.moduleDirRelative = moduleDirRelative;
133  this.moduleDirAbsolute = moduleDirAbsolute;
134  this.archiveDepthCountTree = new ArchiveDepthCountTree();
135  }
136 
145  boolean isSevenZipExtractionSupported(AbstractFile file) {
146  String fileMimeType = fileTypeDetector.getMIMEType(file);
147  for (SupportedArchiveExtractionFormats mimeType : SupportedArchiveExtractionFormats.values()) {
148  if (mimeType.toString().equals(fileMimeType)) {
149  return true;
150  }
151  }
152  return false;
153  }
154 
167  private boolean isZipBombArchiveItemCheck(AbstractFile archiveFile, ISimpleInArchiveItem archiveFileItem) {
168  try {
169  final Long archiveItemSize = archiveFileItem.getSize();
170 
171  //skip the check for small files
172  if (archiveItemSize == null || archiveItemSize < MIN_COMPRESSION_RATIO_SIZE) {
173  return false;
174  }
175 
176  final Long archiveItemPackedSize = archiveFileItem.getPackedSize();
177 
178  if (archiveItemPackedSize == null || archiveItemPackedSize <= 0) {
179  logger.log(Level.WARNING, "Cannot getting compression ratio, cannot detect if zipbomb: {0}, item: {1}", new Object[]{archiveFile.getName(), archiveFileItem.getPath()}); //NON-NLS
180  return false;
181  }
182 
183  int cRatio = (int) (archiveItemSize / archiveItemPackedSize);
184 
185  if (cRatio >= MAX_COMPRESSION_RATIO) {
186  String itemName = archiveFileItem.getPath();
187  logger.log(Level.INFO, "Possible zip bomb detected, compression ration: {0} for in archive item: {1}", new Object[]{cRatio, itemName}); //NON-NLS
188  String msg = NbBundle.getMessage(SevenZipExtractor.class,
189  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg", archiveFile.getName(), itemName);
190  String path;
191  try {
192  path = archiveFile.getUniquePath();
193  } catch (TskCoreException ex) {
194  path = archiveFile.getParentPath() + archiveFile.getName();
195  }
196  String details = NbBundle.getMessage(SevenZipExtractor.class,
197  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails", cRatio, path);
198  //MessageNotifyUtil.Notify.error(msg, details);
199  services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
200  return true;
201  } else {
202  return false;
203  }
204 
205  } catch (SevenZipException ex) {
206  logger.log(Level.WARNING, "Error getting archive item size and cannot detect if zipbomb. ", ex); //NON-NLS
207  return false;
208  }
209  }
210 
219  private ArchiveFormat get7ZipOptions(AbstractFile archiveFile) {
220  // try to get the file type from the BB
221  String detectedFormat = null;
222  detectedFormat = archiveFile.getMIMEType();
223 
224  if (detectedFormat == null) {
225  logger.log(Level.WARNING, "Could not detect format for file: {0}", archiveFile); //NON-NLS
226 
227  // if we don't have attribute info then use file extension
228  String extension = archiveFile.getNameExtension();
229  if ("rar".equals(extension)) //NON-NLS
230  {
231  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
232  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality
233  return RAR;
234  }
235 
236  // Otherwise open the archive using 7zip's built-in auto-detect functionality
237  return null;
238  } else if (detectedFormat.contains("application/x-rar-compressed")) //NON-NLS
239  {
240  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
241  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality
242  return RAR;
243  }
244 
245  // Otherwise open the archive using 7zip's built-in auto-detect functionality
246  return null;
247  }
248 
259  private long getRootArchiveId(AbstractFile file) throws TskCoreException {
260  long id = file.getId();
261  Content parentContent = file.getParent();
262  while (parentContent != null) {
263  id = parentContent.getId();
264  parentContent = parentContent.getParent();
265  }
266  return id;
267  }
268 
283  private List<AbstractFile> getAlreadyExtractedFiles(AbstractFile archiveFile, String archiveFilePath) throws TskCoreException, NoCurrentCaseException {
284  //check if already has derived files, skip
285  //check if local unpacked dir exists
286  if (archiveFile.hasChildren() && new File(moduleDirAbsolute, EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)).exists()) {
287  return Case.getOpenCase().getServices().getFileManager().findFilesByParentPath(getRootArchiveId(archiveFile), archiveFilePath);
288  }
289  return new ArrayList<>();
290  }
291 
299  private String getArchiveFilePath(AbstractFile archiveFile) {
300  try {
301  return archiveFile.getUniquePath();
302  } catch (TskCoreException ex) {
303  return archiveFile.getParentPath() + archiveFile.getName();
304  }
305  }
306 
313  private void makeLocalDirectories(String uniqueArchiveFileName) {
314  final String localRootAbsPath = getLocalRootAbsPath(uniqueArchiveFileName);
315  final File localRoot = new File(localRootAbsPath);
316  if (!localRoot.exists()) {
317  localRoot.mkdirs();
318  }
319  }
320 
333  private String getPathInArchive(ISimpleInArchiveItem item, int itemNumber, AbstractFile archiveFile) throws SevenZipException {
334  String pathInArchive = item.getPath();
335 
336  if (pathInArchive == null || pathInArchive.isEmpty()) {
337  //some formats (.tar.gz) may not be handled correctly -- file in archive has no name/path
338  //handle this for .tar.gz and tgz but assuming the child is tar,
339  //otherwise, unpack using itemNumber as name
340 
341  //TODO this should really be signature based, not extension based
342  String archName = archiveFile.getName();
343  int dotI = archName.lastIndexOf(".");
344  String useName = null;
345  if (dotI != -1) {
346  String base = archName.substring(0, dotI);
347  String ext = archName.substring(dotI);
348  int colonIndex = ext.lastIndexOf(":");
349  if (colonIndex != -1) {
350  // If alternate data stream is found, fix the name
351  // so Windows doesn't choke on the colon character.
352  ext = ext.substring(0, colonIndex);
353  }
354  switch (ext) {
355  case ".gz": //NON-NLS
356  useName = base;
357  break;
358  case ".tgz": //NON-NLS
359  useName = base + ".tar"; //NON-NLS
360  break;
361  case ".bz2": //NON-NLS
362  useName = base;
363  break;
364  }
365  }
366  if (useName == null) {
367  pathInArchive = "/" + archName + "/" + Integer.toString(itemNumber);
368  } else {
369  pathInArchive = "/" + useName;
370  }
371  String msg = NbBundle.getMessage(SevenZipExtractor.class,
372  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.unknownPath.msg",
373  getArchiveFilePath(archiveFile), pathInArchive);
374  logger.log(Level.WARNING, msg);
375  }
376  return pathInArchive;
377  }
378 
379  /*
380  * Get the String that will represent the key for the hashmap which keeps
381  * track of existing files from an AbstractFile
382  */
383  private String getKeyAbstractFile(AbstractFile fileInDatabase) {
384  return fileInDatabase == null ? null : fileInDatabase.getParentPath() + fileInDatabase.getName();
385  }
386 
387  /*
388  * Get the String that will represent the key for the hashmap which keeps
389  * track of existing files from an unpacked node and the archiveFilePath
390  */
391  private String getKeyFromUnpackedNode(UnpackedTree.UnpackedNode node, String archiveFilePath) {
392  return node == null ? null : archiveFilePath + "/" + node.getFileName();
393  }
394 
409  private SevenZipExtractor.UnpackedTree.UnpackedNode unpackNode(ISimpleInArchiveItem item, SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode, String password, long freeDiskSpace, String uniqueExtractedName) throws SevenZipException {
410  //unpack locally if a file
411  final String localAbsPath = moduleDirAbsolute + File.separator + uniqueExtractedName;
412  final String localRelPath = moduleDirRelative + File.separator + uniqueExtractedName;
413  final Date createTime = item.getCreationTime();
414  final Date accessTime = item.getLastAccessTime();
415  final Date writeTime = item.getLastWriteTime();
416  final long createtime = createTime == null ? 0L : createTime.getTime() / 1000;
417  final long modtime = writeTime == null ? 0L : writeTime.getTime() / 1000;
418  final long accesstime = accessTime == null ? 0L : accessTime.getTime() / 1000;
419  SevenZipExtractor.UnpackStream unpackStream = null;
420  boolean isDir = item.isFolder();
421  if (!isDir) {
422  try {
423  // NOTE: item.getSize() may return null in case of certain
424  // archiving formats. Eg: BZ2
425  if (item.getSize() != null) {
426  unpackStream = new SevenZipExtractor.KnownSizeUnpackStream(localAbsPath, item.getSize());
427  } else {
428  unpackStream = new SevenZipExtractor.UnknownSizeUnpackStream(localAbsPath, freeDiskSpace);
429  }
430  ExtractOperationResult result;
431  if (password == null) {
432  result = item.extractSlow(unpackStream);
433  } else {
434  result = item.extractSlow(unpackStream, password);
435  }
436  if (result != ExtractOperationResult.OK) {
437  logger.log(Level.WARNING, "Extraction of : " + localAbsPath + " encountered error " + result); //NON-NLS
438  return null;
439  }
440 
441  } catch (Exception e) {
442  //could be something unexpected with this file, move on
443  logger.log(Level.WARNING, "Could not extract file from archive: " + localAbsPath, e); //NON-NLS
444  } finally {
445  if (unpackStream != null) {
446  //record derived data in unode, to be traversed later after unpacking the archive
447  unpackedNode.addDerivedInfo(unpackStream.getSize(), !isDir,
448  0L, createtime, accesstime, modtime, localRelPath);
449  unpackStream.close();
450  }
451  }
452  } else { // this is a directory, size is always 0
453  unpackedNode.addDerivedInfo(0, !isDir, 0L, createtime, accesstime, modtime, localRelPath);
454  }
455  return unpackedNode;
456  }
457 
465  void unpack(AbstractFile archiveFile) {
466  unpack(archiveFile, null);
467  }
468 
478  @Messages({"SevenZipExtractor.indexError.message=Failed to index encryption detected artifact for keyword search."})
479  boolean unpack(AbstractFile archiveFile, String password) {
480  boolean unpackSuccessful = true; //initialized to true change to false if any files fail to extract and
481  boolean hasEncrypted = false;
482  boolean fullEncryption = true;
483  boolean progressStarted = false;
484  int processedItems = 0;
485  final String archiveFilePath = getArchiveFilePath(archiveFile);
486  final String escapedArchiveFilePath = FileUtil.escapeFileName(archiveFilePath);
487  HashMap<String, ZipFileStatusWrapper> statusMap = new HashMap<>();
488  List<AbstractFile> unpackedFiles = Collections.<AbstractFile>emptyList();
489  ISevenZipInArchive inArchive = null;
490 
491  SevenZipContentReadStream stream = null;
492  final ProgressHandle progress = ProgressHandle.createHandle(Bundle.EmbeddedFileExtractorIngestModule_ArchiveExtractor_moduleName());
493  //recursion depth check for zip bomb
494  final long archiveId = archiveFile.getId();
495  SevenZipExtractor.ArchiveDepthCountTree.Archive parentAr = null;
496  try {
497  blackboard = Case.getOpenCase().getServices().getBlackboard();
498  } catch (NoCurrentCaseException ex) {
499  logger.log(Level.INFO, "Exception while getting open case.", ex); //NON-NLS
500  unpackSuccessful = false;
501  return unpackSuccessful;
502  }
503  try {
504 
505  List<AbstractFile> existingFiles = getAlreadyExtractedFiles(archiveFile, archiveFilePath);
506  for (AbstractFile file : existingFiles) {
507  statusMap.put(getKeyAbstractFile(file), new ZipFileStatusWrapper(file, ZipFileStatus.EXISTS));
508  }
509  } catch (TskCoreException e) {
510  logger.log(Level.INFO, "Error checking if file already has been processed, skipping: {0}", escapedArchiveFilePath); //NON-NLS
511  unpackSuccessful = false;
512  return unpackSuccessful;
513  } catch (NoCurrentCaseException ex) {
514  logger.log(Level.INFO, "No open case was found while trying to unpack the archive file {0}", escapedArchiveFilePath); //NON-NLS
515  unpackSuccessful = false;
516  return unpackSuccessful;
517  }
518  parentAr = archiveDepthCountTree.findArchive(archiveId);
519  if (parentAr == null) {
520  parentAr = archiveDepthCountTree.addArchive(null, archiveId);
521 
522  } else if (parentAr.getDepth() == MAX_DEPTH) {
523  String msg = NbBundle.getMessage(SevenZipExtractor.class,
524  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnMsg.zipBomb", archiveFile.getName());
525  String details = NbBundle.getMessage(SevenZipExtractor.class,
526  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnDetails.zipBomb",
527  parentAr.getDepth(), escapedArchiveFilePath);
528  //MessageNotifyUtil.Notify.error(msg, details);
529  services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
530  unpackSuccessful = false;
531  return unpackSuccessful;
532  }
533  try {
534  stream = new SevenZipContentReadStream(new ReadContentInputStream(archiveFile));
535  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
536  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality.
537  // All other archive formats are still opened using 7zip built-in auto-detect functionality.
538  ArchiveFormat options = get7ZipOptions(archiveFile);
539  if (password == null) {
540  inArchive = SevenZip.openInArchive(options, stream);
541  } else {
542  inArchive = SevenZip.openInArchive(options, stream, password);
543  }
544  int numItems = inArchive.getNumberOfItems();
545  logger.log(Level.INFO, "Count of items in archive: {0}: {1}", new Object[]{escapedArchiveFilePath, numItems}); //NON-NLS
546  progress.start(numItems);
547  progressStarted = true;
548  final ISimpleInArchive simpleInArchive = inArchive.getSimpleInterface();
549 
550  //setup the archive local root folder
551  final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
552  try {
553  makeLocalDirectories(uniqueArchiveFileName);
554  } catch (SecurityException e) {
555  logger.log(Level.SEVERE, "Error setting up output path for archive root: {0}", getLocalRootAbsPath(uniqueArchiveFileName)); //NON-NLS
556  //bail
557  unpackSuccessful = false;
558  return unpackSuccessful;
559  }
560 
561  //initialize tree hierarchy to keep track of unpacked file structure
562  SevenZipExtractor.UnpackedTree unpackedTree = new SevenZipExtractor.UnpackedTree(moduleDirRelative + "/" + uniqueArchiveFileName, archiveFile);
563 
564  long freeDiskSpace;
565  try {
566  freeDiskSpace = services.getFreeDiskSpace();
567  } catch (NullPointerException ex) {
568  //If ingest has not been run at least once getFreeDiskSpace() will throw a null pointer exception
569  //currently getFreeDiskSpace always returns DISK_FREE_SPACE_UNKNOWN
570  freeDiskSpace = IngestMonitor.DISK_FREE_SPACE_UNKNOWN;
571  }
572  //unpack and process every item in archive
573  int itemNumber = 0;
574 
575  for (ISimpleInArchiveItem item : simpleInArchive.getArchiveItems()) {
576  String pathInArchive = getPathInArchive(item, itemNumber, archiveFile);
577 
578  //query for path in db
579  ++itemNumber;
580 
581  //check if possible zip bomb
582  if (isZipBombArchiveItemCheck(archiveFile, item)) {
583  continue; //skip the item
584  }
585  SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode = unpackedTree.addNode(pathInArchive);
586  //update progress bar
587  progress.progress(archiveFile.getName() + ": " + item.getPath(), processedItems);
588 
589  final boolean isEncrypted = item.isEncrypted();
590 
591  if (isEncrypted && password == null) {
592  logger.log(Level.WARNING, "Skipping encrypted file in archive: {0}", pathInArchive); //NON-NLS
593  hasEncrypted = true;
594  unpackSuccessful = false;
595  continue;
596  } else {
597  fullEncryption = false;
598  }
599  // NOTE: item.getSize() may return null in case of certain
600  // archiving formats. Eg: BZ2
601  //check if unpacking this file will result in out of disk space
602  //this is additional to zip bomb prevention mechanism
603  if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && item.getSize() != null && item.getSize() > 0) { //if free space is known and file is not empty.
604  long newDiskSpace = freeDiskSpace - item.getSize();
605  if (newDiskSpace < MIN_FREE_DISK_SPACE) {
606  String msg = NbBundle.getMessage(SevenZipExtractor.class,
607  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.msg",
608  escapedArchiveFilePath, item.getPath());
609  String details = NbBundle.getMessage(SevenZipExtractor.class,
610  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.details");
611  //MessageNotifyUtil.Notify.error(msg, details);
612  services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
613  logger.log(Level.INFO, "Skipping archive item due to insufficient disk space: {0}, {1}", new String[]{escapedArchiveFilePath, item.getPath()}); //NON-NLS
614  logger.log(Level.INFO, "Available disk space: {0}", new Object[]{freeDiskSpace}); //NON-NLS
615  unpackSuccessful = false;
616  continue; //skip this file
617  } else {
618  //update est. disk space during this archive, so we don't need to poll for every file extracted
619  freeDiskSpace = newDiskSpace;
620  }
621  }
622  final String uniqueExtractedName = FileUtil.escapeFileName(uniqueArchiveFileName + File.separator + (item.getItemIndex() / 1000) + File.separator + item.getItemIndex() + "_" + new File(pathInArchive).getName());
623 
624  //create local dirs and empty files before extracted
625  File localFile = new java.io.File(moduleDirAbsolute + File.separator + uniqueExtractedName);
626  //cannot rely on files in top-bottom order
627  if (!localFile.exists()) {
628  try {
629  if (item.isFolder()) {
630  localFile.mkdirs();
631  } else {
632  localFile.getParentFile().mkdirs();
633  try {
634  localFile.createNewFile();
635  } catch (IOException e) {
636  logger.log(Level.SEVERE, "Error creating extracted file: " + localFile.getAbsolutePath(), e); //NON-NLS
637  }
638  }
639  } catch (SecurityException e) {
640  logger.log(Level.SEVERE, "Error setting up output path for unpacked file: {0}", pathInArchive); //NON-NLS
641  //TODO consider bail out / msg to the user
642  }
643  }
644  // skip the rest of this loop if we couldn't create the file
645  if (localFile.exists() == false) {
646  continue;
647  }
648  //find this node in the hierarchy, create if neede;
649  unpackedNode = unpackNode(item, unpackedNode, password,
650  freeDiskSpace, uniqueExtractedName);
651  if (unpackedNode == null) {
652  unpackSuccessful = false;
653  }
654 
655  //update units for progress bar
656  ++processedItems;
657  }
658  // add them to the DB. We wait until the end so that we have the metadata on all of the
659  // intermediate nodes since the order is not guaranteed
660  try {
661  unpackedTree.updateOrAddFileToCaseRec(statusMap, archiveFilePath);
662  unpackedFiles = unpackedTree.getAllFileObjects();
663  //check if children are archives, update archive depth tracking
664  for (AbstractFile unpackedFile : unpackedFiles) {
665  if (unpackedFile == null) {
666  continue;
667  }
668  if (isSevenZipExtractionSupported(unpackedFile)) {
669  archiveDepthCountTree.addArchive(parentAr, unpackedFile.getId());
670  }
671  }
672 
673  } catch (TskCoreException | NoCurrentCaseException e) {
674  logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure", e); //NON-NLS
675  //TODO decide if anything to cleanup, for now bailing
676  }
677 
678  } catch (SevenZipException ex) {
679  logger.log(Level.WARNING, "Error unpacking file: " + archiveFile, ex); //NON-NLS
680  //inbox message
681 
682  // print a message if the file is allocated
683  if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
684  String msg = NbBundle.getMessage(SevenZipExtractor.class,
685  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
686  archiveFile.getName());
687  String details = NbBundle.getMessage(SevenZipExtractor.class,
688  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.details",
689  escapedArchiveFilePath, ex.getMessage());
690  services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
691  }
692  } finally {
693  if (inArchive != null) {
694  try {
695  inArchive.close();
696  } catch (SevenZipException e) {
697  logger.log(Level.SEVERE, "Error closing archive: " + archiveFile, e); //NON-NLS
698  }
699  }
700 
701  if (stream != null) {
702  try {
703  stream.close();
704  } catch (IOException ex) {
705  logger.log(Level.SEVERE, "Error closing stream after unpacking archive: " + archiveFile, ex); //NON-NLS
706  }
707  }
708 
709  //close progress bar
710  if (progressStarted) {
711  progress.finish();
712  }
713  }
714 
715  //create artifact and send user message
716  if (hasEncrypted) {
717  String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
718  try {
719  BlackboardArtifact artifact = archiveFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
720  artifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, EmbeddedFileExtractorModuleFactory.getModuleName(), encryptionType));
721 
722  try {
723  // index the artifact for keyword search
724  blackboard.indexArtifact(artifact);
725  } catch (Blackboard.BlackboardException ex) {
726  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
727  MessageNotifyUtil.Notify.error(
728  Bundle.SevenZipExtractor_indexError_message(), artifact.getDisplayName());
729  }
730 
731  services.fireModuleDataEvent(new ModuleDataEvent(EmbeddedFileExtractorModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
732  } catch (TskCoreException ex) {
733  logger.log(Level.SEVERE, "Error creating blackboard artifact for encryption detected for file: " + escapedArchiveFilePath, ex); //NON-NLS
734  }
735 
736  String msg = NbBundle.getMessage(SevenZipExtractor.class,
737  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
738  String details = NbBundle.getMessage(SevenZipExtractor.class,
739  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.details",
740  archiveFile.getName(), EmbeddedFileExtractorModuleFactory.getModuleName());
741  services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
742  }
743 
744  // adding unpacked extracted derived files to the job after closing relevant resources.
745  if (!unpackedFiles.isEmpty()) {
746  //currently sending a single event for all new files
747  services.fireModuleContentEvent(new ModuleContentEvent(archiveFile));
748  if (context != null) {
749  context.addFilesToJob(unpackedFiles);
750  }
751  }
752  return unpackSuccessful;
753  }
754 
758  private abstract static class UnpackStream implements ISequentialOutStream {
759 
760  private OutputStream output;
761  private String localAbsPath;
762 
763  UnpackStream(String localAbsPath) {
764  this.localAbsPath = localAbsPath;
765  try {
766  output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1);
767  } catch (IOException ex) {
768  logger.log(Level.SEVERE, "Error writing extracted file: " + localAbsPath, ex); //NON-NLS
769  }
770 
771  }
772 
773  public abstract long getSize();
774 
775  OutputStream getOutput() {
776  return output;
777  }
778 
779  String getLocalAbsPath() {
780  return localAbsPath;
781  }
782 
783  public void close() {
784  if (output != null) {
785  try {
786  output.flush();
787  output.close();
788  } catch (IOException e) {
789  logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS
790  }
791  }
792  }
793  }
794 
798  private static class UnknownSizeUnpackStream extends UnpackStream {
799 
800  private long freeDiskSpace;
801  private boolean outOfSpace = false;
802  private long bytesWritten = 0;
803 
804  UnknownSizeUnpackStream(String localAbsPath, long freeDiskSpace) {
805  super(localAbsPath);
806  this.freeDiskSpace = freeDiskSpace;
807  }
808 
809  @Override
810  public long getSize() {
811  return this.bytesWritten;
812  }
813 
814  @Override
815  public int write(byte[] bytes) throws SevenZipException {
816  try {
817  // If the content size is unknown, cautiously write to disk.
818  // Write only if byte array is less than 80% of the current
819  // free disk space.
820  if (freeDiskSpace == IngestMonitor.DISK_FREE_SPACE_UNKNOWN || bytes.length < 0.8 * freeDiskSpace) {
821  getOutput().write(bytes);
822  // NOTE: this method is called multiple times for a
823  // single extractSlow() call. Update bytesWritten and
824  // freeDiskSpace after every write operation.
825  this.bytesWritten += bytes.length;
826  this.freeDiskSpace -= bytes.length;
827  } else {
828  this.outOfSpace = true;
829  logger.log(Level.INFO, NbBundle.getMessage(
830  SevenZipExtractor.class,
831  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
832  throw new SevenZipException(
833  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
834  }
835  } catch (IOException ex) {
836  throw new SevenZipException(
837  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
838  getLocalAbsPath()), ex);
839  }
840  return bytes.length;
841  }
842 
843  @Override
844  public void close() {
845  if (getOutput() != null) {
846  try {
847  getOutput().flush();
848  getOutput().close();
849  if (this.outOfSpace) {
850  Files.delete(Paths.get(getLocalAbsPath()));
851  }
852  } catch (IOException e) {
853  logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", getLocalAbsPath()); //NON-NLS
854  }
855  }
856  }
857  }
858 
862  private static class KnownSizeUnpackStream extends UnpackStream {
863 
864  private long size;
865 
866  KnownSizeUnpackStream(String localAbsPath, long size) {
867  super(localAbsPath);
868  this.size = size;
869  }
870 
871  @Override
872  public long getSize() {
873  return this.size;
874  }
875 
876  @Override
877  public int write(byte[] bytes) throws SevenZipException {
878  try {
879  getOutput().write(bytes);
880  } catch (IOException ex) {
881  throw new SevenZipException(
882  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
883  getLocalAbsPath()), ex);
884  }
885  return bytes.length;
886  }
887  }
888 
896  private class UnpackedTree {
897 
898  final UnpackedNode rootNode;
899 
907  UnpackedTree(String localPathRoot, AbstractFile archiveFile) {
908  this.rootNode = new UnpackedNode();
909  this.rootNode.setFile(archiveFile);
910  this.rootNode.setFileName(archiveFile.getName());
911  this.rootNode.localRelPath = localPathRoot;
912  }
913 
923  UnpackedNode addNode(String filePath) {
924  String[] toks = filePath.split("[\\/\\\\]");
925  List<String> tokens = new ArrayList<>();
926  for (int i = 0; i < toks.length; ++i) {
927  if (!toks[i].isEmpty()) {
928  tokens.add(toks[i]);
929  }
930  }
931  return addNode(rootNode, tokens);
932  }
933 
942  private UnpackedNode addNode(UnpackedNode parent, List<String> tokenPath) {
943  // we found all of the tokens
944  if (tokenPath.isEmpty()) {
945  return parent;
946  }
947 
948  // get the next name in the path and look it up
949  String childName = tokenPath.remove(0);
950  UnpackedNode child = parent.getChild(childName);
951  // create new node
952  if (child == null) {
953  child = new UnpackedNode(childName, parent);
954  }
955 
956  // go down one more level
957  return addNode(child, tokenPath);
958  }
959 
966  List<AbstractFile> getRootFileObjects() {
967  List<AbstractFile> ret = new ArrayList<>();
968  for (UnpackedNode child : rootNode.children) {
969  ret.add(child.getFile());
970  }
971  return ret;
972  }
973 
980  List<AbstractFile> getAllFileObjects() {
981  List<AbstractFile> ret = new ArrayList<>();
982  for (UnpackedNode child : rootNode.children) {
983  getAllFileObjectsRec(ret, child);
984  }
985  return ret;
986  }
987 
988  private void getAllFileObjectsRec(List<AbstractFile> list, UnpackedNode parent) {
989  list.add(parent.getFile());
990  for (UnpackedNode child : parent.children) {
991  getAllFileObjectsRec(list, child);
992  }
993  }
994 
999  void updateOrAddFileToCaseRec(HashMap<String, ZipFileStatusWrapper> statusMap, String archiveFilePath) throws TskCoreException, NoCurrentCaseException {
1000  final FileManager fileManager = Case.getOpenCase().getServices().getFileManager();
1001  for (UnpackedNode child : rootNode.children) {
1002  updateOrAddFileToCaseRec(child, fileManager, statusMap, archiveFilePath);
1003  }
1004  }
1005 
1020  private void updateOrAddFileToCaseRec(UnpackedNode node, FileManager fileManager, HashMap<String, ZipFileStatusWrapper> statusMap, String archiveFilePath) throws TskCoreException {
1021  DerivedFile df;
1022  try {
1023  String nameInDatabase = getKeyFromUnpackedNode(node, archiveFilePath);
1024  ZipFileStatusWrapper existingFile = nameInDatabase == null ? null : statusMap.get(nameInDatabase);
1025  if (existingFile == null) {
1026  df = fileManager.addDerivedFile(node.getFileName(), node.getLocalRelPath(), node.getSize(),
1027  node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
1028  node.isIsFile(), node.getParent().getFile(), "", EmbeddedFileExtractorModuleFactory.getModuleName(),
1029  "", "", TskData.EncodingType.XOR1);
1030  statusMap.put(getKeyAbstractFile(df), new ZipFileStatusWrapper(df, ZipFileStatus.EXISTS));
1031  } else {
1032  String key = getKeyAbstractFile(existingFile.getFile());
1033  if (existingFile.getStatus() == ZipFileStatus.EXISTS && existingFile.getFile().getSize() < node.getSize()) {
1034  existingFile.setStatus(ZipFileStatus.UPDATE);
1035  statusMap.put(key, existingFile);
1036  }
1037  if (existingFile.getStatus() == ZipFileStatus.UPDATE) {
1038  //if the we are updating a file and its mime type was octet-stream we want to re-type it
1039  String mimeType = existingFile.getFile().getMIMEType().equalsIgnoreCase("application/octet-stream") ? null : existingFile.getFile().getMIMEType();
1040  df = fileManager.updateDerivedFile((DerivedFile) existingFile.getFile(), node.getLocalRelPath(), node.getSize(),
1041  node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
1042  node.isIsFile(), mimeType, "", EmbeddedFileExtractorModuleFactory.getModuleName(),
1043  "", "", TskData.EncodingType.XOR1);
1044  } else {
1045  //ALREADY CURRENT - SKIP
1046  statusMap.put(key, new ZipFileStatusWrapper(existingFile.getFile(), ZipFileStatus.SKIP));
1047  df = (DerivedFile) existingFile.getFile();
1048  }
1049  }
1050  node.setFile(df);
1051  } catch (TskCoreException ex) {
1052  logger.log(Level.SEVERE, "Error adding a derived file to db:" + node.getFileName(), ex); //NON-NLS
1053  throw new TskCoreException(
1054  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
1055  node.getFileName()), ex);
1056  }
1057  //recurse adding the children if this file was incomplete the children presumably need to be added
1058  for (UnpackedNode child : node.children) {
1059  updateOrAddFileToCaseRec(child, fileManager, statusMap, getKeyFromUnpackedNode(node, archiveFilePath));
1060  }
1061  }
1062 
1066  private class UnpackedNode {
1067 
1068  private String fileName;
1069  private AbstractFile file;
1070  private List<UnpackedNode> children = new ArrayList<>();
1071  private String localRelPath = "";
1072  private long size;
1073  private long ctime, crtime, atime, mtime;
1074  private boolean isFile;
1076 
1077  //root constructor
1078  UnpackedNode() {
1079  }
1080 
1081  //child node constructor
1082  UnpackedNode(String fileName, UnpackedNode parent) {
1083  this.fileName = fileName;
1084  this.parent = parent;
1085  this.localRelPath = parent.localRelPath + File.separator + fileName;
1086  //new child derived file will be set by unpack() method
1087  parent.children.add(this);
1088  }
1089 
1090  public long getCtime() {
1091  return ctime;
1092  }
1093 
1094  public long getCrtime() {
1095  return crtime;
1096  }
1097 
1098  public long getAtime() {
1099  return atime;
1100  }
1101 
1102  public long getMtime() {
1103  return mtime;
1104  }
1105 
1106  public void setFileName(String fileName) {
1107  this.fileName = fileName;
1108  }
1109 
1110  UnpackedNode getParent() {
1111  return parent;
1112  }
1113 
1114  void addDerivedInfo(long size,
1115  boolean isFile,
1116  long ctime, long crtime, long atime, long mtime, String relLocalPath) {
1117  this.size = size;
1118  this.isFile = isFile;
1119  this.ctime = ctime;
1120  this.crtime = crtime;
1121  this.atime = atime;
1122  this.mtime = mtime;
1123  this.localRelPath = relLocalPath;
1124  }
1125 
1126  void setFile(AbstractFile file) {
1127  this.file = file;
1128  }
1129 
1137  UnpackedNode getChild(String childFileName) {
1138  UnpackedNode ret = null;
1139  for (UnpackedNode child : children) {
1140  if (child.fileName.equals(childFileName)) {
1141  ret = child;
1142  break;
1143  }
1144  }
1145  return ret;
1146  }
1147 
1148  public String getFileName() {
1149  return fileName;
1150  }
1151 
1152  public AbstractFile getFile() {
1153  return file;
1154  }
1155 
1156  public String getLocalRelPath() {
1157  return localRelPath;
1158  }
1159 
1160  public long getSize() {
1161  return size;
1162  }
1163 
1164  public boolean isIsFile() {
1165  return isFile;
1166  }
1167  }
1168  }
1169 
1173  private static class ArchiveDepthCountTree {
1174 
1175  //keeps all nodes refs for easy search
1176  private final List<Archive> archives = new ArrayList<>();
1177 
1185  Archive findArchive(long objectId) {
1186  for (Archive ar : archives) {
1187  if (ar.objectId == objectId) {
1188  return ar;
1189  }
1190  }
1191 
1192  return null;
1193  }
1194 
1203  Archive addArchive(Archive parent, long objectId) {
1204  Archive child = new Archive(parent, objectId);
1205  archives.add(child);
1206  return child;
1207  }
1208 
1209  private static class Archive {
1210 
1211  int depth;
1212  long objectId;
1213  Archive parent;
1214  List<Archive> children;
1215 
1216  Archive(Archive parent, long objectId) {
1217  this.parent = parent;
1218  this.objectId = objectId;
1219  children = new ArrayList<>();
1220  if (parent != null) {
1221  parent.children.add(this);
1222  this.depth = parent.depth + 1;
1223  } else {
1224  this.depth = 0;
1225  }
1226  }
1227 
1233  int getDepth() {
1234  return depth;
1235  }
1236  }
1237  }
1238 
1243  private final class ZipFileStatusWrapper {
1244 
1245  private final AbstractFile abstractFile;
1247 
1255  private ZipFileStatusWrapper(AbstractFile file, ZipFileStatus status) {
1256  abstractFile = file;
1257  zipStatus = status;
1258  }
1259 
1265  private AbstractFile getFile() {
1266  return abstractFile;
1267  }
1268 
1275  return zipStatus;
1276  }
1277 
1283  private void setStatus(ZipFileStatus status) {
1284  zipStatus = status;
1285  }
1286 
1287  }
1288 
1293  private enum ZipFileStatus {
1294  UPDATE, //Should be updated //NON-NLS
1295  SKIP, //File is current can be skipped //NON-NLS
1296  EXISTS //File exists but it is unknown if it is current //NON-NLS
1297  }
1298 }
void updateOrAddFileToCaseRec(UnpackedNode node, FileManager fileManager, HashMap< String, ZipFileStatusWrapper > statusMap, String archiveFilePath)
synchronized DerivedFile addDerivedFile(String fileName, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, Content parentObj, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
UnpackedNode addNode(UnpackedNode parent, List< String > tokenPath)
synchronized DerivedFile updateDerivedFile(DerivedFile derivedFile, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, String mimeType, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)

Copyright © 2012-2016 Basis Technology. Generated on: Mon May 7 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.