Autopsy  4.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
SevenZipExtractor.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2013-2014 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.embeddedfileextractor;
20 
21 import java.io.BufferedOutputStream;
22 import java.io.File;
23 import java.io.FileNotFoundException;
24 import java.io.FileOutputStream;
25 import java.io.IOException;
26 import java.io.OutputStream;
27 import java.nio.file.Files;
28 import java.nio.file.Paths;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.Date;
32 import java.util.List;
33 import java.util.logging.Level;
34 import net.sf.sevenzipjbinding.ArchiveFormat;
35 import static net.sf.sevenzipjbinding.ArchiveFormat.RAR;
36 import net.sf.sevenzipjbinding.ISequentialOutStream;
37 import net.sf.sevenzipjbinding.ISevenZipInArchive;
38 import net.sf.sevenzipjbinding.SevenZip;
39 import net.sf.sevenzipjbinding.SevenZipException;
40 import net.sf.sevenzipjbinding.SevenZipNativeInitializationException;
41 import net.sf.sevenzipjbinding.simple.ISimpleInArchive;
42 import net.sf.sevenzipjbinding.simple.ISimpleInArchiveItem;
43 import org.netbeans.api.progress.ProgressHandle;
44 import org.netbeans.api.progress.ProgressHandleFactory;
45 import org.openide.util.NbBundle;
60 import org.sleuthkit.datamodel.AbstractFile;
61 import org.sleuthkit.datamodel.BlackboardArtifact;
62 import org.sleuthkit.datamodel.BlackboardAttribute;
63 import org.sleuthkit.datamodel.DerivedFile;
64 import org.sleuthkit.datamodel.ReadContentInputStream;
65 import org.sleuthkit.datamodel.TskCoreException;
66 import org.sleuthkit.datamodel.TskData;
67 
68 class SevenZipExtractor {
69 
70  private static final Logger logger = Logger.getLogger(SevenZipExtractor.class.getName());
71  private IngestServices services = IngestServices.getInstance();
72  private final IngestJobContext context;
73  private final FileTypeDetector fileTypeDetector;
74  static final String[] SUPPORTED_EXTENSIONS = {"zip", "rar", "arj", "7z", "7zip", "gzip", "gz", "bzip2", "tar", "tgz",}; // NON-NLS
75  //encryption type strings
76  private static final String ENCRYPTION_FILE_LEVEL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
77  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFileLevel");
78  private static final String ENCRYPTION_FULL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
79  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFull");
80  //zip bomb detection
81  private static final int MAX_DEPTH = 4;
82  private static final int MAX_COMPRESSION_RATIO = 600;
83  private static final long MIN_COMPRESSION_RATIO_SIZE = 500 * 1000000L;
84  private static final long MIN_FREE_DISK_SPACE = 1 * 1000 * 1000000L; //1GB
85  //counts archive depth
86  private ArchiveDepthCountTree archiveDepthCountTree;
87 
88  private String moduleDirRelative;
89  private String moduleDirAbsolute;
90 
91  private Blackboard blackboard;
92 
93  private String getLocalRootAbsPath(String uniqueArchiveFileName) {
94  return moduleDirAbsolute + File.separator + uniqueArchiveFileName;
95  }
96 
101 
102  ZIP("application/zip"), //NON-NLS
103  SEVENZ("application/x-7z-compressed"), //NON-NLS
104  GZIP("application/gzip"), //NON-NLS
105  XGZIP("application/x-gzip"), //NON-NLS
106  XBZIP2("application/x-bzip2"), //NON-NLS
107  XTAR("application/x-tar"), //NON-NLS
108  XGTAR("application/x-gtar"); //NON-NLS
109 
110  private final String mimeType;
111 
112  SupportedArchiveExtractionFormats(final String mimeType) {
113  this.mimeType = mimeType;
114  }
115 
116  @Override
117  public String toString() {
118  return this.mimeType;
119  }
120  // TODO Expand to support more formats after upgrading Tika
121  }
122 
123  SevenZipExtractor(IngestJobContext context, FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute) throws IngestModuleException {
124  if (!SevenZip.isInitializedSuccessfully() && (SevenZip.getLastInitializationException() == null)) {
125  try {
126  SevenZip.initSevenZipFromPlatformJAR();
127  String platform = SevenZip.getUsedPlatform();
128  logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform); //NON-NLS
129  } catch (SevenZipNativeInitializationException e) {
130  logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e); //NON-NLS
131  String msg = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errInitModule.msg",
132  EmbeddedFileExtractorModuleFactory.getModuleName());
133  String details = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.init.errCantInitLib",
134  e.getMessage());
135  services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
136  throw new IngestModuleException(e.getMessage(), e);
137  }
138  }
139  this.context = context;
140  this.fileTypeDetector = fileTypeDetector;
141  this.moduleDirRelative = moduleDirRelative;
142  this.moduleDirAbsolute = moduleDirAbsolute;
143  this.archiveDepthCountTree = new ArchiveDepthCountTree();
144  }
145 
156  boolean isSevenZipExtractionSupported(AbstractFile abstractFile) {
157  try {
158  String abstractFileMimeType = fileTypeDetector.getFileType(abstractFile);
159  for (SupportedArchiveExtractionFormats s : SupportedArchiveExtractionFormats.values()) {
160  if (s.toString().equals(abstractFileMimeType)) {
161  return true;
162  }
163  }
164 
165  return false;
166  } catch (TskCoreException ex) {
167  logger.log(Level.WARNING, "Error executing FileTypeDetector.getFileType()", ex); // NON-NLS
168  }
169 
170  // attempt extension matching
171  final String extension = abstractFile.getNameExtension();
172  for (String supportedExtension : SUPPORTED_EXTENSIONS) {
173  if (extension.equals(supportedExtension)) {
174  return true;
175  }
176  }
177 
178  return false;
179  }
180 
193  private boolean isZipBombArchiveItemCheck(AbstractFile archiveFile, ISimpleInArchiveItem archiveFileItem) {
194  try {
195  final Long archiveItemSize = archiveFileItem.getSize();
196 
197  //skip the check for small files
198  if (archiveItemSize == null || archiveItemSize < MIN_COMPRESSION_RATIO_SIZE) {
199  return false;
200  }
201 
202  final Long archiveItemPackedSize = archiveFileItem.getPackedSize();
203 
204  if (archiveItemPackedSize == null || archiveItemPackedSize <= 0) {
205  logger.log(Level.WARNING, "Cannot getting compression ratio, cannot detect if zipbomb: {0}, item: {1}", new Object[]{archiveFile.getName(), archiveFileItem.getPath()}); //NON-NLS
206  return false;
207  }
208 
209  int cRatio = (int) (archiveItemSize / archiveItemPackedSize);
210 
211  if (cRatio >= MAX_COMPRESSION_RATIO) {
212  String itemName = archiveFileItem.getPath();
213  logger.log(Level.INFO, "Possible zip bomb detected, compression ration: {0} for in archive item: {1}", new Object[]{cRatio, itemName}); //NON-NLS
214  String msg = NbBundle.getMessage(SevenZipExtractor.class,
215  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg", archiveFile.getName(), itemName);
216  String path;
217  try {
218  path = archiveFile.getUniquePath();
219  } catch (TskCoreException ex) {
220  path = archiveFile.getParentPath() + archiveFile.getName();
221  }
222  String details = NbBundle.getMessage(SevenZipExtractor.class,
223  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails", cRatio, path);
224  //MessageNotifyUtil.Notify.error(msg, details);
225  services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
226  return true;
227  } else {
228  return false;
229  }
230 
231  } catch (SevenZipException ex) {
232  logger.log(Level.WARNING, "Error getting archive item size and cannot detect if zipbomb. ", ex); //NON-NLS
233  return false;
234  }
235  }
236 
245  private ArchiveFormat get7ZipOptions(AbstractFile archiveFile) {
246  // try to get the file type from the BB
247  String detectedFormat = null;
248  detectedFormat = archiveFile.getMIMEType();
249 
250  if (detectedFormat == null) {
251  logger.log(Level.WARNING, "Could not detect format for file: {0}", archiveFile); //NON-NLS
252 
253  // if we don't have attribute info then use file extension
254  String extension = archiveFile.getNameExtension();
255  if ("rar".equals(extension)) //NON-NLS
256  {
257  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
258  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality
259  return RAR;
260  }
261 
262  // Otherwise open the archive using 7zip's built-in auto-detect functionality
263  return null;
264  } else if (detectedFormat.contains("application/x-rar-compressed")) //NON-NLS
265  {
266  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
267  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality
268  return RAR;
269  }
270 
271  // Otherwise open the archive using 7zip's built-in auto-detect functionality
272  return null;
273  }
274 
283  void unpack(AbstractFile archiveFile) {
284  blackboard = Case.getCurrentCase().getServices().getBlackboard();
285  String archiveFilePath;
286  try {
287  archiveFilePath = archiveFile.getUniquePath();
288  } catch (TskCoreException ex) {
289  archiveFilePath = archiveFile.getParentPath() + archiveFile.getName();
290  }
291 
292  //check if already has derived files, skip
293  try {
294  if (archiveFile.hasChildren()) {
295  //check if local unpacked dir exists
296  if (new File(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)).exists()) {
297  logger.log(Level.INFO, "File already has been processed as it has children and local unpacked file, skipping: {0}", archiveFilePath); //NON-NLS
298  return;
299  }
300  }
301  } catch (TskCoreException e) {
302  logger.log(Level.INFO, "Error checking if file already has been processed, skipping: {0}", archiveFilePath); //NON-NLS
303  return;
304  }
305 
306  List<AbstractFile> unpackedFiles = Collections.<AbstractFile>emptyList();
307 
308  //recursion depth check for zip bomb
309  final long archiveId = archiveFile.getId();
310  SevenZipExtractor.ArchiveDepthCountTree.Archive parentAr = archiveDepthCountTree.findArchive(archiveId);
311  if (parentAr == null) {
312  parentAr = archiveDepthCountTree.addArchive(null, archiveId);
313  } else if (parentAr.getDepth() == MAX_DEPTH) {
314  String msg = NbBundle.getMessage(SevenZipExtractor.class,
315  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnMsg.zipBomb", archiveFile.getName());
316  String details = NbBundle.getMessage(SevenZipExtractor.class,
317  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnDetails.zipBomb",
318  parentAr.getDepth(), archiveFilePath);
319  //MessageNotifyUtil.Notify.error(msg, details);
320  services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
321  return;
322  }
323 
324  boolean hasEncrypted = false;
325  boolean fullEncryption = true;
326 
327  ISevenZipInArchive inArchive = null;
328  SevenZipContentReadStream stream = null;
329 
330  final ProgressHandle progress = ProgressHandleFactory.createHandle(
331  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.moduleName"));
332  int processedItems = 0;
333 
334  boolean progressStarted = false;
335  try {
336  stream = new SevenZipContentReadStream(new ReadContentInputStream(archiveFile));
337 
338  // for RAR files we need to open them explicitly as RAR. Otherwise, if there is a ZIP archive inside RAR archive
339  // it will be opened incorrectly when using 7zip's built-in auto-detect functionality.
340  // All other archive formats are still opened using 7zip built-in auto-detect functionality.
341  ArchiveFormat options = get7ZipOptions(archiveFile);
342  inArchive = SevenZip.openInArchive(options, stream);
343 
344  int numItems = inArchive.getNumberOfItems();
345  logger.log(Level.INFO, "Count of items in archive: {0}: {1}", new Object[]{archiveFilePath, numItems}); //NON-NLS
346  progress.start(numItems);
347  progressStarted = true;
348 
349  final ISimpleInArchive simpleInArchive = inArchive.getSimpleInterface();
350 
351  //setup the archive local root folder
352  final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
353  final String localRootAbsPath = getLocalRootAbsPath(uniqueArchiveFileName);
354  final File localRoot = new File(localRootAbsPath);
355  if (!localRoot.exists()) {
356  try {
357  localRoot.mkdirs();
358  } catch (SecurityException e) {
359  logger.log(Level.SEVERE, "Error setting up output path for archive root: {0}", localRootAbsPath); //NON-NLS
360  //bail
361  return;
362  }
363  }
364 
365  //initialize tree hierarchy to keep track of unpacked file structure
366  SevenZipExtractor.UnpackedTree unpackedTree = new SevenZipExtractor.UnpackedTree(moduleDirRelative + "/" + uniqueArchiveFileName, archiveFile);
367 
368  long freeDiskSpace = services.getFreeDiskSpace();
369 
370  //unpack and process every item in archive
371  int itemNumber = 0;
372  for (ISimpleInArchiveItem item : simpleInArchive.getArchiveItems()) {
373  String pathInArchive = item.getPath();
374 
375  if (pathInArchive == null || pathInArchive.isEmpty()) {
376  //some formats (.tar.gz) may not be handled correctly -- file in archive has no name/path
377  //handle this for .tar.gz and tgz but assuming the child is tar,
378  //otherwise, unpack using itemNumber as name
379 
380  //TODO this should really be signature based, not extension based
381  String archName = archiveFile.getName();
382  int dotI = archName.lastIndexOf(".");
383  String useName = null;
384  if (dotI != -1) {
385  String base = archName.substring(0, dotI);
386  String ext = archName.substring(dotI);
387  int colonIndex = ext.lastIndexOf(":");
388  if (colonIndex != -1) {
389  // If alternate data stream is found, fix the name
390  // so Windows doesn't choke on the colon character.
391  ext = ext.substring(0, colonIndex);
392  }
393  switch (ext) {
394  case ".gz": //NON-NLS
395  useName = base;
396  break;
397  case ".tgz": //NON-NLS
398  useName = base + ".tar"; //NON-NLS
399  break;
400  case ".bz2": //NON-NLS
401  useName = base;
402  break;
403  }
404  }
405 
406  if (useName == null) {
407  pathInArchive = "/" + archName + "/" + Integer.toString(itemNumber);
408  } else {
409  pathInArchive = "/" + useName;
410  }
411 
412  String msg = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.unknownPath.msg",
413  archiveFilePath, pathInArchive);
414  logger.log(Level.WARNING, msg);
415 
416  }
417  archiveFilePath = FileUtil.escapeFileName(archiveFilePath);
418  ++itemNumber;
419 
420  //check if possible zip bomb
421  if (isZipBombArchiveItemCheck(archiveFile, item)) {
422  continue; //skip the item
423  }
424 
425  //find this node in the hierarchy, create if needed
426  SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode = unpackedTree.addNode(pathInArchive);
427 
428  String fileName = unpackedNode.getFileName();
429 
430  //update progress bar
431  progress.progress(archiveFile.getName() + ": " + fileName, processedItems);
432 
433  final boolean isEncrypted = item.isEncrypted();
434  final boolean isDir = item.isFolder();
435 
436  if (isEncrypted) {
437  logger.log(Level.WARNING, "Skipping encrypted file in archive: {0}", pathInArchive); //NON-NLS
438  hasEncrypted = true;
439  continue;
440  } else {
441  fullEncryption = false;
442  }
443 
444  // NOTE: item.getSize() may return null in case of certain
445  // archiving formats. Eg: BZ2
446  Long size = item.getSize();
447 
448  //check if unpacking this file will result in out of disk space
449  //this is additional to zip bomb prevention mechanism
450  if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && size != null && size > 0) { //if free space is known and file is not empty.
451  long newDiskSpace = freeDiskSpace - size;
452  if (newDiskSpace < MIN_FREE_DISK_SPACE) {
453  String msg = NbBundle.getMessage(SevenZipExtractor.class,
454  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.msg",
455  archiveFilePath, fileName);
456  String details = NbBundle.getMessage(SevenZipExtractor.class,
457  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.details");
458  //MessageNotifyUtil.Notify.error(msg, details);
459  services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
460  logger.log(Level.INFO, "Skipping archive item due to insufficient disk space: {0}, {1}", new Object[]{archiveFilePath, fileName}); //NON-NLS
461  logger.log(Level.INFO, "Available disk space: {0}", new Object[]{freeDiskSpace}); //NON-NLS
462  continue; //skip this file
463  } else {
464  //update est. disk space during this archive, so we don't need to poll for every file extracted
465  freeDiskSpace = newDiskSpace;
466  }
467  }
468 
469  final String uniqueExtractedName = FileUtil.escapeFileName(uniqueArchiveFileName + File.separator + (item.getItemIndex() / 1000) + File.separator + item.getItemIndex() + "_" + new File(pathInArchive).getName());
470 
471  //final String localRelPath = unpackDir + File.separator + localFileRelPath;
472  final String localRelPath = moduleDirRelative + File.separator + uniqueExtractedName;
473  final String localAbsPath = moduleDirAbsolute + File.separator + uniqueExtractedName;
474 
475  //create local dirs and empty files before extracted
476  File localFile = new java.io.File(localAbsPath);
477  //cannot rely on files in top-bottom order
478  if (!localFile.exists()) {
479  try {
480  if (isDir) {
481  localFile.mkdirs();
482  } else {
483  localFile.getParentFile().mkdirs();
484  try {
485  localFile.createNewFile();
486  } catch (IOException e) {
487  logger.log(Level.SEVERE, "Error creating extracted file: " + localFile.getAbsolutePath(), e); //NON-NLS
488  }
489  }
490  } catch (SecurityException e) {
491  logger.log(Level.SEVERE, "Error setting up output path for unpacked file: {0}", pathInArchive); //NON-NLS
492  //TODO consider bail out / msg to the user
493  }
494  }
495 
496  // skip the rest of this loop if we couldn't create the file
497  if (localFile.exists() == false) {
498  continue;
499  }
500 
501  final Date createTime = item.getCreationTime();
502  final Date accessTime = item.getLastAccessTime();
503  final Date writeTime = item.getLastWriteTime();
504  final long createtime = createTime == null ? 0L : createTime.getTime() / 1000;
505  final long modtime = writeTime == null ? 0L : writeTime.getTime() / 1000;
506  final long accesstime = accessTime == null ? 0L : accessTime.getTime() / 1000;
507 
508  //unpack locally if a file
509  SevenZipExtractor.UnpackStream unpackStream = null;
510  if (!isDir) {
511  try {
512  if (size != null) {
513  unpackStream = new SevenZipExtractor.KnownSizeUnpackStream(localAbsPath, size);
514  } else {
515  unpackStream = new SevenZipExtractor.UnknownSizeUnpackStream(localAbsPath, freeDiskSpace);
516  }
517  item.extractSlow(unpackStream);
518  } catch (Exception e) {
519  //could be something unexpected with this file, move on
520  logger.log(Level.WARNING, "Could not extract file from archive: " + localAbsPath, e); //NON-NLS
521  } finally {
522  if (unpackStream != null) {
523  //record derived data in unode, to be traversed later after unpacking the archive
524  unpackedNode.addDerivedInfo(unpackStream.getSize(), !isDir,
525  0L, createtime, accesstime, modtime, localRelPath);
526  unpackStream.close();
527  }
528  }
529  } else { // this is a directory, size is always 0
530  unpackedNode.addDerivedInfo(0, !isDir,
531  0L, createtime, accesstime, modtime, localRelPath);
532  }
533 
534  //update units for progress bar
535  ++processedItems;
536  }
537 
538  // add them to the DB. We wait until the end so that we have the metadata on all of the
539  // intermediate nodes since the order is not guaranteed
540  try {
541  unpackedTree.addDerivedFilesToCase();
542  unpackedFiles = unpackedTree.getAllFileObjects();
543 
544  //check if children are archives, update archive depth tracking
545  for (AbstractFile unpackedFile : unpackedFiles) {
546  if (isSevenZipExtractionSupported(unpackedFile)) {
547  archiveDepthCountTree.addArchive(parentAr, unpackedFile.getId());
548  }
549  }
550 
551  } catch (TskCoreException e) {
552  logger.log(Level.SEVERE, "Error populating complete derived file hierarchy from the unpacked dir structure"); //NON-NLS
553  //TODO decide if anything to cleanup, for now bailing
554  }
555 
556  } catch (SevenZipException ex) {
557  logger.log(Level.WARNING, "Error unpacking file: {0}", archiveFile); //NON-NLS
558  //inbox message
559 
560  // print a message if the file is allocated
561  if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
562  String msg = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
563  archiveFile.getName());
564  String details = NbBundle.getMessage(SevenZipExtractor.class,
565  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.details",
566  archiveFilePath, ex.getMessage());
567  services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
568  }
569  } finally {
570  if (inArchive != null) {
571  try {
572  inArchive.close();
573  } catch (SevenZipException e) {
574  logger.log(Level.SEVERE, "Error closing archive: " + archiveFile, e); //NON-NLS
575  }
576  }
577 
578  if (stream != null) {
579  try {
580  stream.close();
581  } catch (IOException ex) {
582  logger.log(Level.SEVERE, "Error closing stream after unpacking archive: " + archiveFile, ex); //NON-NLS
583  }
584  }
585 
586  //close progress bar
587  if (progressStarted) {
588  progress.finish();
589  }
590  }
591 
592  //create artifact and send user message
593  if (hasEncrypted) {
594  String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
595  try {
596  BlackboardArtifact artifact = archiveFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
597  artifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, EmbeddedFileExtractorModuleFactory.getModuleName(), encryptionType));
598 
599  try {
600  // index the artifact for keyword search
601  blackboard.indexArtifact(artifact);
602  } catch (Blackboard.BlackboardException ex) {
603  logger.log(Level.SEVERE, NbBundle.getMessage(Blackboard.class, "Blackboard.unableToIndexArtifact.error.msg", artifact.getDisplayName()), ex); //NON-NLS
604  MessageNotifyUtil.Notify.error(
605  NbBundle.getMessage(Blackboard.class, "Blackboard.unableToIndexArtifact.exception.msg"), artifact.getDisplayName());
606  }
607 
608  services.fireModuleDataEvent(new ModuleDataEvent(EmbeddedFileExtractorModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
609  } catch (TskCoreException ex) {
610  logger.log(Level.SEVERE, "Error creating blackboard artifact for encryption detected for file: " + archiveFilePath, ex); //NON-NLS
611  }
612 
613  String msg = NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
614  String details = NbBundle.getMessage(SevenZipExtractor.class,
615  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.details",
616  archiveFile.getName(), EmbeddedFileExtractorModuleFactory.getModuleName());
617  services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
618  }
619 
620  // adding unpacked extracted derived files to the job after closing relevant resources.
621  if (!unpackedFiles.isEmpty()) {
622  //currently sending a single event for all new files
623  services.fireModuleContentEvent(new ModuleContentEvent(archiveFile));
624  context.addFilesToJob(unpackedFiles);
625  }
626  }
627 
631  private abstract static class UnpackStream implements ISequentialOutStream {
632 
633  private OutputStream output;
634  private String localAbsPath;
635 
636  UnpackStream(String localAbsPath) {
637  this.localAbsPath = localAbsPath;
638  try {
639  output = new BufferedOutputStream(new FileOutputStream(localAbsPath));
640  } catch (FileNotFoundException ex) {
641  logger.log(Level.SEVERE, "Error writing extracted file: " + localAbsPath, ex); //NON-NLS
642  }
643 
644  }
645 
646  public abstract long getSize();
647 
648  OutputStream getOutput() {
649  return output;
650  }
651 
652  String getLocalAbsPath() {
653  return localAbsPath;
654  }
655 
656  public void close() {
657  if (output != null) {
658  try {
659  output.flush();
660  output.close();
661  } catch (IOException e) {
662  logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS
663  }
664  }
665  }
666  }
667 
671  private static class UnknownSizeUnpackStream extends UnpackStream {
672 
673  private long freeDiskSpace;
674  private boolean outOfSpace = false;
675  private long bytesWritten = 0;
676 
677  UnknownSizeUnpackStream(String localAbsPath, long freeDiskSpace) {
678  super(localAbsPath);
679  this.freeDiskSpace = freeDiskSpace;
680  }
681 
682  @Override
683  public long getSize() {
684  return this.bytesWritten;
685  }
686 
687  @Override
688  public int write(byte[] bytes) throws SevenZipException {
689  try {
690  // If the content size is unknown, cautiously write to disk.
691  // Write only if byte array is less than 80% of the current
692  // free disk space.
693  if (freeDiskSpace == IngestMonitor.DISK_FREE_SPACE_UNKNOWN || bytes.length < 0.8 * freeDiskSpace) {
694  getOutput().write(bytes);
695  // NOTE: this method is called multiple times for a
696  // single extractSlow() call. Update bytesWritten and
697  // freeDiskSpace after every write operation.
698  this.bytesWritten += bytes.length;
699  this.freeDiskSpace -= bytes.length;
700  } else {
701  this.outOfSpace = true;
702  logger.log(Level.INFO, NbBundle.getMessage(
703  SevenZipExtractor.class,
704  "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
705  throw new SevenZipException(
706  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
707  }
708  } catch (IOException ex) {
709  throw new SevenZipException(
710  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
711  getLocalAbsPath()), ex);
712  }
713  return bytes.length;
714  }
715 
716  @Override
717  public void close() {
718  if (getOutput() != null) {
719  try {
720  getOutput().flush();
721  getOutput().close();
722  if (this.outOfSpace) {
723  Files.delete(Paths.get(getLocalAbsPath()));
724  }
725  } catch (IOException e) {
726  logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", getLocalAbsPath()); //NON-NLS
727  }
728  }
729  }
730  }
731 
735  private static class KnownSizeUnpackStream extends UnpackStream {
736 
737  private long size;
738 
739  KnownSizeUnpackStream(String localAbsPath, long size) {
740  super(localAbsPath);
741  this.size = size;
742  }
743 
744  @Override
745  public long getSize() {
746  return this.size;
747  }
748 
749  @Override
750  public int write(byte[] bytes) throws SevenZipException {
751  try {
752  getOutput().write(bytes);
753  } catch (IOException ex) {
754  throw new SevenZipException(
755  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
756  getLocalAbsPath()), ex);
757  }
758  return bytes.length;
759  }
760  }
761 
769  private class UnpackedTree {
770 
771  final UnpackedNode rootNode;
772 
780  UnpackedTree(String localPathRoot, AbstractFile archiveFile) {
781  this.rootNode = new UnpackedNode();
782  this.rootNode.setFile(archiveFile);
783  this.rootNode.setFileName(archiveFile.getName());
784  this.rootNode.localRelPath = localPathRoot;
785  }
786 
796  UnpackedNode addNode(String filePath) {
797  String[] toks = filePath.split("[\\/\\\\]");
798  List<String> tokens = new ArrayList<>();
799  for (int i = 0; i < toks.length; ++i) {
800  if (!toks[i].isEmpty()) {
801  tokens.add(toks[i]);
802  }
803  }
804  return addNode(rootNode, tokens);
805  }
806 
814  private UnpackedNode addNode(UnpackedNode parent, List<String> tokenPath) {
815  // we found all of the tokens
816  if (tokenPath.isEmpty()) {
817  return parent;
818  }
819 
820  // get the next name in the path and look it up
821  String childName = tokenPath.remove(0);
822  UnpackedNode child = parent.getChild(childName);
823  // create new node
824  if (child == null) {
825  child = new UnpackedNode(childName, parent);
826  }
827 
828  // go down one more level
829  return addNode(child, tokenPath);
830  }
831 
838  List<AbstractFile> getRootFileObjects() {
839  List<AbstractFile> ret = new ArrayList<>();
840  for (UnpackedNode child : rootNode.children) {
841  ret.add(child.getFile());
842  }
843  return ret;
844  }
845 
852  List<AbstractFile> getAllFileObjects() {
853  List<AbstractFile> ret = new ArrayList<>();
854  for (UnpackedNode child : rootNode.children) {
855  getAllFileObjectsRec(ret, child);
856  }
857  return ret;
858  }
859 
860  private void getAllFileObjectsRec(List<AbstractFile> list, UnpackedNode parent) {
861  list.add(parent.getFile());
862  for (UnpackedNode child : parent.children) {
863  getAllFileObjectsRec(list, child);
864  }
865  }
866 
871  void addDerivedFilesToCase() throws TskCoreException {
872  final FileManager fileManager = Case.getCurrentCase().getServices().getFileManager();
873  for (UnpackedNode child : rootNode.children) {
874  addDerivedFilesToCaseRec(child, fileManager);
875  }
876  }
877 
878  private void addDerivedFilesToCaseRec(UnpackedNode node, FileManager fileManager) throws TskCoreException {
879  final String fileName = node.getFileName();
880 
881  try {
882  DerivedFile df = fileManager.addDerivedFile(fileName, node.getLocalRelPath(), node.getSize(),
883  node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
884  node.isIsFile(), node.getParent().getFile(), "", EmbeddedFileExtractorModuleFactory.getModuleName(), "", "");
885  node.setFile(df);
886 
887  } catch (TskCoreException ex) {
888  logger.log(Level.SEVERE, "Error adding a derived file to db:" + fileName, ex); //NON-NLS
889  throw new TskCoreException(
890  NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
891  fileName), ex);
892  }
893 
894  //recurse
895  for (UnpackedNode child : node.children) {
896  addDerivedFilesToCaseRec(child, fileManager);
897  }
898  }
899 
903  private class UnpackedNode {
904 
905  private String fileName;
906  private AbstractFile file;
907  private List<UnpackedNode> children = new ArrayList<>();
908  private String localRelPath = "";
909  private long size;
910  private long ctime, crtime, atime, mtime;
911  private boolean isFile;
913 
914  //root constructor
915  UnpackedNode() {
916  }
917 
918  //child node constructor
919  UnpackedNode(String fileName, UnpackedNode parent) {
920  this.fileName = fileName;
921  this.parent = parent;
922  this.localRelPath = parent.localRelPath + File.separator + fileName;
923  //new child derived file will be set by unpack() method
924  parent.children.add(this);
925 
926  }
927 
928  public long getCtime() {
929  return ctime;
930  }
931 
932  public long getCrtime() {
933  return crtime;
934  }
935 
936  public long getAtime() {
937  return atime;
938  }
939 
940  public long getMtime() {
941  return mtime;
942  }
943 
944  public void setFileName(String fileName) {
945  this.fileName = fileName;
946  }
947 
948  UnpackedNode getParent() {
949  return parent;
950  }
951 
952  void addDerivedInfo(long size,
953  boolean isFile,
954  long ctime, long crtime, long atime, long mtime, String relLocalPath) {
955  this.size = size;
956  this.isFile = isFile;
957  this.ctime = ctime;
958  this.crtime = crtime;
959  this.atime = atime;
960  this.mtime = mtime;
961  this.localRelPath = relLocalPath;
962  }
963 
964  void setFile(AbstractFile file) {
965  this.file = file;
966  }
967 
975  UnpackedNode getChild(String childFileName) {
976  UnpackedNode ret = null;
977  for (UnpackedNode child : children) {
978  if (child.fileName.equals(childFileName)) {
979  ret = child;
980  break;
981  }
982  }
983  return ret;
984  }
985 
986  public String getFileName() {
987  return fileName;
988  }
989 
990  public AbstractFile getFile() {
991  return file;
992  }
993 
994  public String getLocalRelPath() {
995  return localRelPath;
996  }
997 
998  public long getSize() {
999  return size;
1000  }
1001 
1002  public boolean isIsFile() {
1003  return isFile;
1004  }
1005  }
1006  }
1007 
1011  private static class ArchiveDepthCountTree {
1012 
1013  //keeps all nodes refs for easy search
1014  private final List<Archive> archives = new ArrayList<>();
1015 
1023  Archive findArchive(long objectId) {
1024  for (Archive ar : archives) {
1025  if (ar.objectId == objectId) {
1026  return ar;
1027  }
1028  }
1029 
1030  return null;
1031  }
1032 
1041  Archive addArchive(Archive parent, long objectId) {
1042  Archive child = new Archive(parent, objectId);
1043  archives.add(child);
1044  return child;
1045  }
1046 
1047  private static class Archive {
1048 
1049  int depth;
1050  long objectId;
1051  Archive parent;
1052  List<Archive> children;
1053 
1054  Archive(Archive parent, long objectId) {
1055  this.parent = parent;
1056  this.objectId = objectId;
1057  children = new ArrayList<>();
1058  if (parent != null) {
1059  parent.children.add(this);
1060  this.depth = parent.depth + 1;
1061  } else {
1062  this.depth = 0;
1063  }
1064  }
1065 
1071  int getDepth() {
1072  return depth;
1073  }
1074  }
1075  }
1076 
1077 }
UnpackedNode addNode(UnpackedNode parent, List< String > tokenPath)
synchronized DerivedFile addDerivedFile(String fileName, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, AbstractFile parentFile, String rederiveDetails, String toolName, String toolVersion, String otherDetails)

Copyright © 2012-2015 Basis Technology. Generated on: Wed Apr 6 2016
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.