19 package org.sleuthkit.autopsy.modules.embeddedfileextractor;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.io.OutputStream;
25 import java.nio.file.Files;
26 import java.nio.file.Paths;
27 import java.util.ArrayList;
28 import java.util.Collections;
29 import java.util.Date;
30 import java.util.List;
31 import java.util.logging.Level;
32 import net.sf.sevenzipjbinding.ArchiveFormat;
33 import static net.sf.sevenzipjbinding.ArchiveFormat.RAR;
34 import net.sf.sevenzipjbinding.ISequentialOutStream;
35 import net.sf.sevenzipjbinding.ISevenZipInArchive;
36 import net.sf.sevenzipjbinding.SevenZip;
37 import net.sf.sevenzipjbinding.SevenZipException;
38 import net.sf.sevenzipjbinding.SevenZipNativeInitializationException;
39 import net.sf.sevenzipjbinding.simple.ISimpleInArchive;
40 import net.sf.sevenzipjbinding.simple.ISimpleInArchiveItem;
41 import org.netbeans.api.progress.ProgressHandle;
42 import org.openide.util.NbBundle;
43 import org.openide.util.NbBundle.Messages;
66 class SevenZipExtractor {
68 private static final Logger logger = Logger.getLogger(SevenZipExtractor.class.getName());
69 private IngestServices services = IngestServices.getInstance();
70 private final IngestJobContext context;
71 private final FileTypeDetector fileTypeDetector;
72 static final String[] SUPPORTED_EXTENSIONS = {
"zip",
"rar",
"arj",
"7z",
"7zip",
"gzip",
"gz",
"bzip2",
"tar",
"tgz",};
74 private static final String ENCRYPTION_FILE_LEVEL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
75 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFileLevel");
76 private static final String ENCRYPTION_FULL = NbBundle.getMessage(EmbeddedFileExtractorIngestModule.class,
77 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.encryptionFull");
79 private static final int MAX_DEPTH = 4;
80 private static final int MAX_COMPRESSION_RATIO = 600;
81 private static final long MIN_COMPRESSION_RATIO_SIZE = 500 * 1000000L;
82 private static final long MIN_FREE_DISK_SPACE = 1 * 1000 * 1000000L;
84 private ArchiveDepthCountTree archiveDepthCountTree;
86 private String moduleDirRelative;
87 private String moduleDirAbsolute;
89 private Blackboard blackboard;
91 private String getLocalRootAbsPath(String uniqueArchiveFileName) {
92 return moduleDirAbsolute + File.separator + uniqueArchiveFileName;
107 XRAR(
"application/x-rar-compressed");
112 this.mimeType = mimeType;
117 return this.mimeType;
122 SevenZipExtractor(
IngestJobContext context,
FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute)
throws SevenZipNativeInitializationException {
123 if (!SevenZip.isInitializedSuccessfully() && (SevenZip.getLastInitializationException() == null)) {
124 SevenZip.initSevenZipFromPlatformJAR();
126 this.context = context;
127 this.fileTypeDetector = fileTypeDetector;
128 this.moduleDirRelative = moduleDirRelative;
129 this.moduleDirAbsolute = moduleDirAbsolute;
130 this.archiveDepthCountTree =
new ArchiveDepthCountTree();
141 boolean isSevenZipExtractionSupported(AbstractFile abstractFile) {
142 String abstractFileMimeType = fileTypeDetector.getMIMEType(abstractFile);
143 for (SupportedArchiveExtractionFormats s : SupportedArchiveExtractionFormats.values()) {
144 if (s.toString().equals(abstractFileMimeType)) {
163 private boolean isZipBombArchiveItemCheck(AbstractFile archiveFile, ISimpleInArchiveItem archiveFileItem) {
165 final Long archiveItemSize = archiveFileItem.getSize();
168 if (archiveItemSize == null || archiveItemSize < MIN_COMPRESSION_RATIO_SIZE) {
172 final Long archiveItemPackedSize = archiveFileItem.getPackedSize();
174 if (archiveItemPackedSize == null || archiveItemPackedSize <= 0) {
175 logger.log(Level.WARNING,
"Cannot getting compression ratio, cannot detect if zipbomb: {0}, item: {1}",
new Object[]{archiveFile.getName(), archiveFileItem.getPath()});
179 int cRatio = (int) (archiveItemSize / archiveItemPackedSize);
181 if (cRatio >= MAX_COMPRESSION_RATIO) {
182 String itemName = archiveFileItem.getPath();
183 logger.log(Level.INFO,
"Possible zip bomb detected, compression ration: {0} for in archive item: {1}",
new Object[]{cRatio, itemName});
184 String msg = NbBundle.getMessage(SevenZipExtractor.class,
185 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnMsg", archiveFile.getName(), itemName);
188 path = archiveFile.getUniquePath();
189 }
catch (TskCoreException ex) {
190 path = archiveFile.getParentPath() + archiveFile.getName();
192 String details = NbBundle.getMessage(SevenZipExtractor.class,
193 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.isZipBombCheck.warnDetails", cRatio, path);
195 services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
201 }
catch (SevenZipException ex) {
202 logger.log(Level.WARNING,
"Error getting archive item size and cannot detect if zipbomb. ", ex);
215 private ArchiveFormat get7ZipOptions(AbstractFile archiveFile) {
217 String detectedFormat = null;
218 detectedFormat = archiveFile.getMIMEType();
220 if (detectedFormat == null) {
221 logger.log(Level.WARNING,
"Could not detect format for file: {0}", archiveFile);
224 String extension = archiveFile.getNameExtension();
225 if (
"rar".equals(extension))
234 }
else if (detectedFormat.contains(
"application/x-rar-compressed"))
253 @Messages({
"SevenZipExtractor.indexError.message=Failed to index encryption detected artifact for keyword search."})
254 void unpack(AbstractFile archiveFile) {
255 blackboard = Case.getCurrentCase().getServices().getBlackboard();
256 String archiveFilePath;
258 archiveFilePath = archiveFile.getUniquePath();
259 }
catch (TskCoreException ex) {
260 archiveFilePath = archiveFile.getParentPath() + archiveFile.getName();
265 if (archiveFile.hasChildren()) {
267 if (
new File(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)).exists()) {
268 logger.log(Level.INFO,
"File already has been processed as it has children and local unpacked file, skipping: {0}", archiveFilePath);
272 }
catch (TskCoreException e) {
273 logger.log(Level.INFO,
"Error checking if file already has been processed, skipping: {0}", archiveFilePath);
277 List<AbstractFile> unpackedFiles = Collections.<AbstractFile>emptyList();
280 final long archiveId = archiveFile.getId();
281 SevenZipExtractor.ArchiveDepthCountTree.Archive parentAr = archiveDepthCountTree.findArchive(archiveId);
282 if (parentAr == null) {
283 parentAr = archiveDepthCountTree.addArchive(null, archiveId);
284 }
else if (parentAr.getDepth() == MAX_DEPTH) {
285 String msg = NbBundle.getMessage(SevenZipExtractor.class,
286 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnMsg.zipBomb", archiveFile.getName());
287 String details = NbBundle.getMessage(SevenZipExtractor.class,
288 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnDetails.zipBomb",
289 parentAr.getDepth(), archiveFilePath);
291 services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
295 boolean hasEncrypted =
false;
296 boolean fullEncryption =
true;
298 ISevenZipInArchive inArchive = null;
299 SevenZipContentReadStream stream = null;
301 final ProgressHandle progress = ProgressHandle.createHandle(Bundle.EmbeddedFileExtractorIngestModule_ArchiveExtractor_moduleName());
302 int processedItems = 0;
304 boolean progressStarted =
false;
306 stream =
new SevenZipContentReadStream(
new ReadContentInputStream(archiveFile));
311 ArchiveFormat options = get7ZipOptions(archiveFile);
312 inArchive = SevenZip.openInArchive(options, stream);
314 int numItems = inArchive.getNumberOfItems();
315 logger.log(Level.INFO,
"Count of items in archive: {0}: {1}",
new Object[]{archiveFilePath, numItems});
316 progress.start(numItems);
317 progressStarted =
true;
319 final ISimpleInArchive simpleInArchive = inArchive.getSimpleInterface();
322 final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
323 final String localRootAbsPath = getLocalRootAbsPath(uniqueArchiveFileName);
324 final File localRoot =
new File(localRootAbsPath);
325 if (!localRoot.exists()) {
328 }
catch (SecurityException e) {
329 logger.log(Level.SEVERE,
"Error setting up output path for archive root: {0}", localRootAbsPath);
336 SevenZipExtractor.UnpackedTree unpackedTree =
new SevenZipExtractor.UnpackedTree(moduleDirRelative +
"/" + uniqueArchiveFileName, archiveFile);
338 long freeDiskSpace = services.getFreeDiskSpace();
342 for (ISimpleInArchiveItem item : simpleInArchive.getArchiveItems()) {
343 String pathInArchive = item.getPath();
345 if (pathInArchive == null || pathInArchive.isEmpty()) {
351 String archName = archiveFile.getName();
352 int dotI = archName.lastIndexOf(
".");
353 String useName = null;
355 String base = archName.substring(0, dotI);
356 String ext = archName.substring(dotI);
357 int colonIndex = ext.lastIndexOf(
":");
358 if (colonIndex != -1) {
361 ext = ext.substring(0, colonIndex);
368 useName = base +
".tar";
376 if (useName == null) {
377 pathInArchive =
"/" + archName +
"/" + Integer.toString(itemNumber);
379 pathInArchive =
"/" + useName;
382 String msg = NbBundle.getMessage(SevenZipExtractor.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.unknownPath.msg",
383 archiveFilePath, pathInArchive);
384 logger.log(Level.WARNING, msg);
387 archiveFilePath = FileUtil.escapeFileName(archiveFilePath);
391 if (isZipBombArchiveItemCheck(archiveFile, item)) {
396 SevenZipExtractor.UnpackedTree.UnpackedNode unpackedNode = unpackedTree.addNode(pathInArchive);
398 String fileName = unpackedNode.getFileName();
401 progress.progress(archiveFile.getName() +
": " + fileName, processedItems);
403 final boolean isEncrypted = item.isEncrypted();
404 final boolean isDir = item.isFolder();
407 logger.log(Level.WARNING,
"Skipping encrypted file in archive: {0}", pathInArchive);
411 fullEncryption =
false;
416 Long size = item.getSize();
420 if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && size != null && size > 0) {
421 long newDiskSpace = freeDiskSpace - size;
422 if (newDiskSpace < MIN_FREE_DISK_SPACE) {
423 String msg = NbBundle.getMessage(SevenZipExtractor.class,
424 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.msg",
425 archiveFilePath, fileName);
426 String details = NbBundle.getMessage(SevenZipExtractor.class,
427 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.notEnoughDiskSpace.details");
429 services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
430 logger.log(Level.INFO,
"Skipping archive item due to insufficient disk space: {0}, {1}",
new Object[]{archiveFilePath, fileName});
431 logger.log(Level.INFO,
"Available disk space: {0}",
new Object[]{freeDiskSpace});
435 freeDiskSpace = newDiskSpace;
439 final String uniqueExtractedName = FileUtil.escapeFileName(uniqueArchiveFileName + File.separator + (item.getItemIndex() / 1000) + File.separator + item.getItemIndex() +
"_" +
new File(pathInArchive).getName());
442 final String localRelPath = moduleDirRelative + File.separator + uniqueExtractedName;
443 final String localAbsPath = moduleDirAbsolute + File.separator + uniqueExtractedName;
446 File localFile =
new java.io.File(localAbsPath);
448 if (!localFile.exists()) {
453 localFile.getParentFile().mkdirs();
455 localFile.createNewFile();
456 }
catch (IOException e) {
457 logger.log(Level.SEVERE,
"Error creating extracted file: " + localFile.getAbsolutePath(), e);
460 }
catch (SecurityException e) {
461 logger.log(Level.SEVERE,
"Error setting up output path for unpacked file: {0}", pathInArchive);
467 if (localFile.exists() ==
false) {
471 final Date createTime = item.getCreationTime();
472 final Date accessTime = item.getLastAccessTime();
473 final Date writeTime = item.getLastWriteTime();
474 final long createtime = createTime == null ? 0L : createTime.getTime() / 1000;
475 final long modtime = writeTime == null ? 0L : writeTime.getTime() / 1000;
476 final long accesstime = accessTime == null ? 0L : accessTime.getTime() / 1000;
479 SevenZipExtractor.UnpackStream unpackStream = null;
483 unpackStream =
new SevenZipExtractor.KnownSizeUnpackStream(localAbsPath, size);
485 unpackStream =
new SevenZipExtractor.UnknownSizeUnpackStream(localAbsPath, freeDiskSpace);
487 item.extractSlow(unpackStream);
488 }
catch (Exception e) {
490 logger.log(Level.WARNING,
"Could not extract file from archive: " + localAbsPath, e);
492 if (unpackStream != null) {
494 unpackedNode.addDerivedInfo(unpackStream.getSize(), !isDir,
495 0L, createtime, accesstime, modtime, localRelPath);
496 unpackStream.close();
500 unpackedNode.addDerivedInfo(0, !isDir,
501 0L, createtime, accesstime, modtime, localRelPath);
511 unpackedTree.addDerivedFilesToCase();
512 unpackedFiles = unpackedTree.getAllFileObjects();
515 for (AbstractFile unpackedFile : unpackedFiles) {
516 if (isSevenZipExtractionSupported(unpackedFile)) {
517 archiveDepthCountTree.addArchive(parentAr, unpackedFile.getId());
521 }
catch (TskCoreException e) {
522 logger.log(Level.SEVERE,
"Error populating complete derived file hierarchy from the unpacked dir structure");
526 }
catch (SevenZipException ex) {
527 logger.log(Level.WARNING,
"Error unpacking file: {0}", archiveFile);
531 if (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) {
532 String msg = NbBundle.getMessage(SevenZipExtractor.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.msg",
533 archiveFile.getName());
534 String details = NbBundle.getMessage(SevenZipExtractor.class,
535 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.errUnpacking.details",
536 archiveFilePath, ex.getMessage());
537 services.postMessage(IngestMessage.createErrorMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
540 if (inArchive != null) {
543 }
catch (SevenZipException e) {
544 logger.log(Level.SEVERE,
"Error closing archive: " + archiveFile, e);
548 if (stream != null) {
551 }
catch (IOException ex) {
552 logger.log(Level.SEVERE,
"Error closing stream after unpacking archive: " + archiveFile, ex);
557 if (progressStarted) {
564 String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
566 BlackboardArtifact artifact = archiveFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
567 artifact.addAttribute(
new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, EmbeddedFileExtractorModuleFactory.getModuleName(), encryptionType));
571 blackboard.indexArtifact(artifact);
572 }
catch (Blackboard.BlackboardException ex) {
573 logger.log(Level.SEVERE,
"Unable to index blackboard artifact " + artifact.getArtifactID(), ex);
574 MessageNotifyUtil.Notify.error(
575 Bundle.SevenZipExtractor_indexError_message(), artifact.getDisplayName());
578 services.fireModuleDataEvent(
new ModuleDataEvent(EmbeddedFileExtractorModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
579 }
catch (TskCoreException ex) {
580 logger.log(Level.SEVERE,
"Error creating blackboard artifact for encryption detected for file: " + archiveFilePath, ex);
583 String msg = NbBundle.getMessage(SevenZipExtractor.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.msg");
584 String details = NbBundle.getMessage(SevenZipExtractor.class,
585 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.encrFileDetected.details",
586 archiveFile.getName(), EmbeddedFileExtractorModuleFactory.getModuleName());
587 services.postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), msg, details));
591 if (!unpackedFiles.isEmpty()) {
593 services.fireModuleContentEvent(
new ModuleContentEvent(archiveFile));
594 context.addFilesToJob(unpackedFiles);
601 private abstract static class UnpackStream implements ISequentialOutStream {
609 output =
new EncodedFileOutputStream(
new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1);
610 }
catch (IOException ex) {
611 logger.log(Level.SEVERE,
"Error writing extracted file: " + localAbsPath, ex);
616 public abstract long getSize();
618 OutputStream getOutput() {
622 String getLocalAbsPath() {
627 if (output != null) {
631 }
catch (IOException e) {
632 logger.log(Level.SEVERE,
"Error closing unpack stream for file: {0}", localAbsPath);
658 public int write(byte[] bytes)
throws SevenZipException {
664 getOutput().write(bytes);
668 this.bytesWritten += bytes.length;
669 this.freeDiskSpace -= bytes.length;
671 this.outOfSpace =
true;
672 logger.log(Level.INFO, NbBundle.getMessage(
673 SevenZipExtractor.class,
674 "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
675 throw new SevenZipException(
676 NbBundle.getMessage(SevenZipExtractor.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg"));
678 }
catch (IOException ex) {
679 throw new SevenZipException(
680 NbBundle.getMessage(SevenZipExtractor.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
681 getLocalAbsPath()), ex);
688 if (getOutput() != null) {
692 if (this.outOfSpace) {
693 Files.delete(Paths.get(getLocalAbsPath()));
695 }
catch (IOException e) {
696 logger.log(Level.SEVERE,
"Error closing unpack stream for file: {0}", getLocalAbsPath());
720 public int write(byte[] bytes)
throws SevenZipException {
722 getOutput().write(bytes);
723 }
catch (IOException ex) {
724 throw new SevenZipException(
725 NbBundle.getMessage(SevenZipExtractor.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg",
726 getLocalAbsPath()), ex);
750 UnpackedTree(String localPathRoot, AbstractFile archiveFile) {
752 this.rootNode.setFile(archiveFile);
767 String[] toks = filePath.split(
"[\\/\\\\]");
768 List<String> tokens =
new ArrayList<>();
769 for (
int i = 0; i < toks.length; ++i) {
770 if (!toks[i].isEmpty()) {
774 return addNode(rootNode, tokens);
787 if (tokenPath.isEmpty()) {
792 String childName = tokenPath.remove(0);
800 return addNode(child, tokenPath);
809 List<AbstractFile> getRootFileObjects() {
810 List<AbstractFile> ret =
new ArrayList<>();
811 for (UnpackedNode child : rootNode.
children) {
812 ret.add(child.getFile());
823 List<AbstractFile> getAllFileObjects() {
824 List<AbstractFile> ret =
new ArrayList<>();
825 for (UnpackedNode child : rootNode.
children) {
842 void addDerivedFilesToCase() throws TskCoreException {
844 for (UnpackedNode child : rootNode.
children) {
850 final String fileName = node.getFileName();
853 DerivedFile df = fileManager.
addDerivedFile(fileName, node.getLocalRelPath(), node.getSize(),
854 node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
856 "",
"", TskData.EncodingType.XOR1);
859 }
catch (TskCoreException ex) {
860 logger.log(Level.SEVERE,
"Error adding a derived file to db:" + fileName, ex);
861 throw new TskCoreException(
862 NbBundle.getMessage(SevenZipExtractor.class,
"EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackedTree.exception.msg",
879 private List<UnpackedNode>
children =
new ArrayList<>();
880 private String localRelPath =
"";
882 private long ctime, crtime, atime, mtime;
924 void addDerivedInfo(
long size,
926 long ctime,
long crtime,
long atime,
long mtime, String relLocalPath) {
930 this.crtime = crtime;
933 this.localRelPath = relLocalPath;
936 void setFile(AbstractFile file) {
947 UnpackedNode getChild(String childFileName) {
948 UnpackedNode ret = null;
949 for (UnpackedNode child : children) {
950 if (child.fileName.equals(childFileName)) {
986 private final List<Archive>
archives =
new ArrayList<>();
995 Archive findArchive(
long objectId) {
997 if (ar.objectId == objectId) {
1013 Archive addArchive(Archive parent,
long objectId) {
1014 Archive child =
new Archive(parent, objectId);
1015 archives.add(child);
1024 List<Archive> children;
1027 this.parent = parent;
1028 this.objectId = objectId;
1029 children =
new ArrayList<>();
1030 if (parent != null) {
1031 parent.children.add(
this);
1032 this.depth = parent.depth + 1;
FileManager getFileManager()
synchronized DerivedFile addDerivedFile(String fileName, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, Content parentObj, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
static final int DISK_FREE_SPACE_UNKNOWN
static Case getCurrentCase()