Autopsy  4.9.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
ThunderbirdMboxFileIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2014 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.thunderbirdparser;
20 
21 import java.io.File;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.HashSet;
26 import java.util.List;
27 import java.util.Set;
28 import java.util.logging.Level;
29 import java.util.regex.Matcher;
30 import java.util.regex.Pattern;
31 import org.openide.util.NbBundle;
32 import org.openide.util.NbBundle.Messages;
48 import org.sleuthkit.datamodel.AbstractFile;
49 import org.sleuthkit.datamodel.Account;
50 import org.sleuthkit.datamodel.AccountFileInstance;
51 import org.sleuthkit.datamodel.BlackboardArtifact;
52 import org.sleuthkit.datamodel.BlackboardAttribute;
53 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
54 import org.sleuthkit.datamodel.DerivedFile;
55 import org.sleuthkit.datamodel.Relationship;
56 import org.sleuthkit.datamodel.TskCoreException;
57 import org.sleuthkit.datamodel.TskData;
58 import org.sleuthkit.datamodel.TskDataException;
59 import org.sleuthkit.datamodel.TskException;
60 
66 public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
67 
68  private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName());
73 
75  }
76 
77  @Override
78  @Messages ({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."})
79  public void startUp(IngestJobContext context) throws IngestModuleException {
80  this.context = context;
81  try {
83  } catch (NoCurrentCaseException ex) {
84  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
85  throw new IngestModuleException(Bundle.ThunderbirdMboxFileIngestModule_noOpenCase_errMsg(), ex);
86  }
87  }
88 
89  @Override
90  public ProcessResult process(AbstractFile abstractFile) {
91 
92  try {
94  } catch (NoCurrentCaseException ex) {
95  logger.log(Level.SEVERE, "Exception while getting open case.", ex);
96  return ProcessResult.ERROR;
97  }
98 
99  // skip known
100  if (abstractFile.getKnown().equals(TskData.FileKnown.KNOWN)) {
101  return ProcessResult.OK;
102  }
103 
104  //skip unalloc
105  if ((abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) ||
106  (abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
107  return ProcessResult.OK;
108  }
109 
110  if ((abstractFile.isFile() == false)) {
111  return ProcessResult.OK;
112  }
113 
114  // check its signature
115  boolean isMbox = false;
116  try {
117  byte[] t = new byte[64];
118  if (abstractFile.getSize() > 64) {
119  int byteRead = abstractFile.read(t, 0, 64);
120  if (byteRead > 0) {
121  isMbox = MboxParser.isValidMimeTypeMbox(t);
122  }
123  }
124  } catch (TskException ex) {
125  logger.log(Level.WARNING, null, ex);
126  }
127 
128  if (isMbox) {
129  return processMBox(abstractFile);
130  }
131 
132  if (PstParser.isPstFile(abstractFile)) {
133  return processPst(abstractFile);
134  }
135 
136  return ProcessResult.OK;
137  }
138 
146  @Messages({"ThunderbirdMboxFileIngestModule.processPst.indexError.message=Failed to index encryption detected artifact for keyword search."})
147  private ProcessResult processPst(AbstractFile abstractFile) {
148  String fileName;
149  try {
150  fileName = getTempPath() + File.separator + abstractFile.getName()
151  + "-" + String.valueOf(abstractFile.getId());
152  } catch (NoCurrentCaseException ex) {
153  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
154  return ProcessResult.ERROR;
155  }
156  File file = new File(fileName);
157 
158  long freeSpace = services.getFreeDiskSpace();
159  if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (abstractFile.getSize() >= freeSpace)) {
160  logger.log(Level.WARNING, "Not enough disk space to write file to disk."); //NON-NLS
162  NbBundle.getMessage(this.getClass(),
163  "ThunderbirdMboxFileIngestModule.processPst.errMsg.outOfDiskSpace",
164  abstractFile.getName()));
165  services.postMessage(msg);
166  return ProcessResult.OK;
167  }
168 
169  try {
170  ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled);
171  } catch (IOException ex) {
172  logger.log(Level.WARNING, "Failed writing pst file to disk.", ex); //NON-NLS
173  return ProcessResult.OK;
174  }
175 
176  PstParser parser = new PstParser(services);
177  PstParser.ParseResult result = parser.parse(file, abstractFile.getId());
178 
179  if (result == PstParser.ParseResult.OK) {
180  try {
181  // parse success: Process email and add artifacts
182  processEmails(parser.getResults(), abstractFile);
183  } catch (NoCurrentCaseException ex) {
184  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
185  return ProcessResult.ERROR;
186  }
187 
188  } else if (result == PstParser.ParseResult.ENCRYPT) {
189  // encrypted pst: Add encrypted file artifact
190  try {
191  BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
192  artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel")));
193 
194  try {
195  // index the artifact for keyword search
196  blackboard.indexArtifact(artifact);
197  } catch (Blackboard.BlackboardException ex) {
198  MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName());
199  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
200  }
201 
202  services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
203  } catch (TskCoreException ex) {
204  logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS
205  }
206  } else {
207  // parsing error: log message
208  postErrorMessage(
209  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg",
210  abstractFile.getName()),
211  NbBundle.getMessage(this.getClass(),
212  "ThunderbirdMboxFileIngestModule.processPst.errProcFile.details"));
213  logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS
214  return ProcessResult.ERROR;
215  }
216 
217  if (file.delete() == false) {
218  logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS
219  }
220 
221  String errors = parser.getErrors();
222  if (errors.isEmpty() == false) {
223  postErrorMessage(
224  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processPst.errProcFile.msg2",
225  abstractFile.getName()), errors);
226  }
227 
228  return ProcessResult.OK;
229  }
230 
238  private ProcessResult processMBox(AbstractFile abstractFile) {
239  String mboxFileName = abstractFile.getName();
240  String mboxParentDir = abstractFile.getParentPath();
241  // use the local path to determine the e-mail folder structure
242  String emailFolder = "";
243  // email folder is everything after "Mail" or ImapMail
244  if (mboxParentDir.contains("/Mail/")) { //NON-NLS
245  emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/Mail/") + 5); //NON-NLS
246  } else if (mboxParentDir.contains("/ImapMail/")) { //NON-NLS
247  emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/ImapMail/") + 9); //NON-NLS
248  }
249  emailFolder = emailFolder + mboxFileName;
250  emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS
251 
252  String fileName;
253  try {
254  fileName = getTempPath() + File.separator + abstractFile.getName()
255  + "-" + String.valueOf(abstractFile.getId());
256  } catch (NoCurrentCaseException ex) {
257  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
258  return ProcessResult.ERROR;
259  }
260  File file = new File(fileName);
261 
262  long freeSpace = services.getFreeDiskSpace();
263  if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (abstractFile.getSize() >= freeSpace)) {
264  logger.log(Level.WARNING, "Not enough disk space to write file to disk."); //NON-NLS
265  postErrorMessage(
266  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processMBox.errProcFile.msg",
267  abstractFile.getName()),
268  NbBundle.getMessage(this.getClass(),
269  "ThunderbirdMboxFileIngestModule.processMBox.errProfFile.details"));
270  return ProcessResult.OK;
271  }
272 
273  try {
274  ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled);
275  } catch (IOException ex) {
276  logger.log(Level.WARNING, "Failed writing mbox file to disk.", ex); //NON-NLS
277  return ProcessResult.OK;
278  }
279 
280  MboxParser parser = new MboxParser(services, emailFolder);
281  List<EmailMessage> emails = parser.parse(file, abstractFile.getId());
282  try {
283  processEmails(emails, abstractFile);
284  } catch (NoCurrentCaseException ex) {
285  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
286  return ProcessResult.ERROR;
287  }
288 
289  if (file.delete() == false) {
290  logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS
291  }
292 
293  String errors = parser.getErrors();
294  if (errors.isEmpty() == false) {
295  postErrorMessage(
296  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.processMBox.errProcFile.msg2",
297  abstractFile.getName()), errors);
298  }
299 
300  return ProcessResult.OK;
301  }
302 
309  static String getTempPath() throws NoCurrentCaseException {
310  String tmpDir = Case.getCurrentCaseThrows().getTempDirectory() + File.separator
311  + "EmailParser"; //NON-NLS
312  File dir = new File(tmpDir);
313  if (dir.exists() == false) {
314  dir.mkdirs();
315  }
316  return tmpDir;
317  }
318 
325  static String getModuleOutputPath() throws NoCurrentCaseException {
326  String outDir = Case.getCurrentCaseThrows().getModuleDirectory() + File.separator
327  + EmailParserModuleFactory.getModuleName();
328  File dir = new File(outDir);
329  if (dir.exists() == false) {
330  dir.mkdirs();
331  }
332  return outDir;
333  }
334 
341  static String getRelModuleOutputPath() throws NoCurrentCaseException {
342  return Case.getCurrentCaseThrows().getModuleOutputDirectoryRelativePath() + File.separator
343  + EmailParserModuleFactory.getModuleName();
344  }
345 
354  private void processEmails(List<EmailMessage> emails, AbstractFile abstractFile) throws NoCurrentCaseException {
355  List<AbstractFile> derivedFiles = new ArrayList<>();
356 
357 
358 
359  for (EmailMessage email : emails) {
360  BlackboardArtifact msgArtifact = addArtifact(email, abstractFile);
361 
362  if ((msgArtifact != null) && (email.hasAttachment())) {
363  derivedFiles.addAll(handleAttachments(email.getAttachments(), abstractFile, msgArtifact ));
364  }
365  }
366 
367  if (derivedFiles.isEmpty() == false) {
368  for (AbstractFile derived : derivedFiles) {
369  services.fireModuleContentEvent(new ModuleContentEvent(derived));
370  }
371  }
372  context.addFilesToJob(derivedFiles);
373  services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
374  }
375 
386  private List<AbstractFile> handleAttachments(List<EmailMessage.Attachment> attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) {
387  List<AbstractFile> files = new ArrayList<>();
388  for (EmailMessage.Attachment attach : attachments) {
389  String filename = attach.getName();
390  long crTime = attach.getCrTime();
391  long mTime = attach.getmTime();
392  long aTime = attach.getaTime();
393  long cTime = attach.getcTime();
394  String relPath = attach.getLocalPath();
395  long size = attach.getSize();
396  TskData.EncodingType encodingType = attach.getEncodingType();
397 
398  try {
399  DerivedFile df = fileManager.addDerivedFile(filename, relPath,
400  size, cTime, crTime, aTime, mTime, true, messageArtifact, "",
401  EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType);
402  files.add(df);
403  } catch (TskCoreException ex) {
404  postErrorMessage(
405  NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.errMsg",
406  abstractFile.getName()),
407  NbBundle.getMessage(this.getClass(),
408  "ThunderbirdMboxFileIngestModule.handleAttch.errMsg.details", filename));
409  logger.log(Level.INFO, "", ex);
410  }
411  }
412  return files;
413  }
414 
422  private Set<String> findEmailAddresess(String input) {
423  Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b",
424  Pattern.CASE_INSENSITIVE);
425  Matcher m = p.matcher(input);
426  Set<String> emailAddresses = new HashSet<String>();
427  while (m.find()) {
428  emailAddresses.add( m.group());
429  }
430  return emailAddresses;
431  }
432 
440  @Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."})
441  private BlackboardArtifact addArtifact(EmailMessage email, AbstractFile abstractFile) throws NoCurrentCaseException {
442  BlackboardArtifact bbart = null;
443  List<BlackboardAttribute> bbattributes = new ArrayList<>();
444  String to = email.getRecipients();
445  String cc = email.getCc();
446  String bcc = email.getBcc();
447  String from = email.getSender();
448  long dateL = email.getSentDate();
449  String headers = email.getHeaders();
450  String body = email.getTextBody();
451  String bodyHTML = email.getHtmlBody();
452  String rtf = email.getRtfBody();
453  String subject = email.getSubject();
454  long id = email.getId();
455  String localPath = email.getLocalPath();
456 
457  List<String> senderAddressList = new ArrayList<>();
458  String senderAddress;
459  senderAddressList.addAll(findEmailAddresess(from));
460 
461  AccountFileInstance senderAccountInstance = null;
462 
463  Case openCase = Case.getCurrentCaseThrows();
464 
465  if (senderAddressList.size() == 1) {
466  senderAddress = senderAddressList.get(0);
467  try {
468  senderAccountInstance = openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile);
469  }
470  catch(TskCoreException ex) {
471  logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS
472  }
473  }
474  else {
475  logger.log(Level.WARNING, "Failed to find sender address, from = "+ from); //NON-NLS
476  }
477 
478  List<String> recipientAddresses = new ArrayList<>();
479  recipientAddresses.addAll(findEmailAddresess(to));
480  recipientAddresses.addAll(findEmailAddresess(cc));
481  recipientAddresses.addAll(findEmailAddresess(bcc));
482 
483  List<AccountFileInstance> recipientAccountInstances = new ArrayList<>();
484  recipientAddresses.forEach((addr) -> {
485  try {
486  AccountFileInstance recipientAccountInstance =
487  openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr,
488  EmailParserModuleFactory.getModuleName(), abstractFile);
489  recipientAccountInstances.add(recipientAccountInstance);
490  }
491  catch(TskCoreException ex) {
492  logger.log(Level.WARNING, "Failed to create account for email address " + addr, ex); //NON-NLS
493  }
494  });
495 
496  addEmailAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes);
497  addEmailAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes);
498  addEmailAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes);
499  addEmailAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes);
500 
501  addEmailAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes);
502  addEmailAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes);
503 
504  addEmailAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes);
505 
506  addEmailAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)),
507  ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes);
508 
509  addEmailAttribute(((localPath.isEmpty() == false) ? localPath : "/foo/bar"),
510  ATTRIBUTE_TYPE.TSK_PATH, bbattributes);
511 
512  addEmailAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes);
513  addEmailAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes);
514  addEmailAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes);
515 
516 
517  try {
518 
519  bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG);
520  bbart.addAttributes(bbattributes);
521 
522  // Add account relationships
523  openCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL);
524 
525  try {
526  // index the artifact for keyword search
527  blackboard.indexArtifact(bbart);
528  } catch (Blackboard.BlackboardException ex) {
529  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + bbart.getArtifactID(), ex); //NON-NLS
530  MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_addArtifact_indexError_message(), bbart.getDisplayName());
531  }
532  } catch (TskCoreException | TskDataException ex) {
533  logger.log(Level.WARNING, null, ex);
534  }
535 
536  return bbart;
537  }
538 
539  private void addEmailAttribute(String stringVal, ATTRIBUTE_TYPE attrType, Collection<BlackboardAttribute> bbattributes) {
540  if (stringVal.isEmpty() == false) {
541  bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal));
542  }
543  }
544  private void addEmailAttribute(long longVal, ATTRIBUTE_TYPE attrType, Collection<BlackboardAttribute> bbattributes) {
545  if (longVal > 0) {
546  bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), longVal));
547  }
548  }
549 
550  void postErrorMessage(String subj, String details) {
551  IngestMessage ingestMessage = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleVersion(), subj, details);
552  services.postMessage(ingestMessage);
553  }
554 
555  IngestServices getServices() {
556  return services;
557  }
558 
559  @Override
560  public void shutDown() {
561  // nothing to shut down
562  }
563 }
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
void addEmailAttribute(String stringVal, ATTRIBUTE_TYPE attrType, Collection< BlackboardAttribute > bbattributes)
BlackboardArtifact addArtifact(EmailMessage email, AbstractFile abstractFile)
static< T > long writeToFile(Content content, java.io.File outputFile, ProgressHandle progress, Future< T > worker, boolean source)
synchronized DerivedFile addDerivedFile(String fileName, String localPath, long size, long ctime, long crtime, long atime, long mtime, boolean isFile, Content parentObj, String rederiveDetails, String toolName, String toolVersion, String otherDetails, TskData.EncodingType encodingType)
void addEmailAttribute(long longVal, ATTRIBUTE_TYPE attrType, Collection< BlackboardAttribute > bbattributes)
void processEmails(List< EmailMessage > emails, AbstractFile abstractFile)
void addFilesToJob(List< AbstractFile > files)
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
void fireModuleContentEvent(ModuleContentEvent moduleContentEvent)
static void error(String title, String message)
synchronized void indexArtifact(BlackboardArtifact artifact)
Definition: Blackboard.java:58
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
List< AbstractFile > handleAttachments(List< EmailMessage.Attachment > attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact)
static synchronized IngestServices getInstance()

Copyright © 2012-2018 Basis Technology. Generated on: Tue Dec 18 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.