Autopsy  4.14.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
XRYMessagesFileParser.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2019-2020 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.datasourceprocessors.xry;
20 
21 import java.io.IOException;
22 import java.nio.file.Path;
23 import java.time.format.DateTimeParseException;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.HashSet;
27 import java.util.List;
28 import java.util.Objects;
29 import java.util.Optional;
30 import java.util.Set;
31 import java.util.logging.Level;
33 import org.sleuthkit.datamodel.Account;
34 import org.sleuthkit.datamodel.Blackboard.BlackboardException;
35 import org.sleuthkit.datamodel.BlackboardAttribute;
36 import org.sleuthkit.datamodel.Content;
37 import org.sleuthkit.datamodel.SleuthkitCase;
38 import org.sleuthkit.datamodel.TskCoreException;
39 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
40 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CommunicationDirection;
41 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.MessageReadStatus;
42 
46 final class XRYMessagesFileParser implements XRYFileParser {
47 
48  private static final Logger logger = Logger.getLogger(
49  XRYMessagesFileParser.class.getName());
50 
51  private static final String PARSER_NAME = "XRY DSP";
52 
57  private enum XryKey {
58  DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED),
59  DIRECTION("direction", null),
60  MESSAGE("message", null),
61  NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON),
62  TEXT("text", null),
63  TIME("time", null),
64  SERVICE_CENTER("service center", null),
65  FROM("from", null),
66  TO("to", null),
67  //The following keys either need special processing or more time and data to find a type.
68  STORAGE("storage", null),
69  NUMBER("number", null),
70  TYPE("type", null),
71  TEL("tel", null),
72  FOLDER("folder", null),
73  NAME("name", null),
74  INDEX("index", null),
75  STATUS("status", null);
76 
77  private final String name;
78  private final BlackboardAttribute.ATTRIBUTE_TYPE type;
79 
80  XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) {
81  this.name = name;
82  this.type = type;
83  }
84 
85  public BlackboardAttribute.ATTRIBUTE_TYPE getType() {
86  return type;
87  }
88 
89  public String getDisplayName() {
90  return name;
91  }
92 
99  public static boolean contains(String name) {
100  try {
101  XryKey.fromDisplayName(name);
102  return true;
103  } catch (IllegalArgumentException ex) {
104  return false;
105  }
106  }
107 
118  public static XryKey fromDisplayName(String name) {
119  String normalizedName = name.trim().toLowerCase();
120  for (XryKey keyChoice : XryKey.values()) {
121  if (normalizedName.equals(keyChoice.name)) {
122  return keyChoice;
123  }
124  }
125 
126  throw new IllegalArgumentException(String.format("Key [ %s ] was not found."
127  + " All keys should be tested with contains.", name));
128  }
129  }
130 
134  private enum XryNamespace {
135  FROM("from"),
136  PARTICIPANT("participant"),
137  TO("to"),
138  NONE(null);
139 
140  private final String name;
141 
142  XryNamespace(String name) {
143  this.name = name;
144  }
145 
153  public static boolean contains(String xryNamespace) {
154  try {
155  XryNamespace.fromDisplayName(xryNamespace);
156  return true;
157  } catch (IllegalArgumentException ex) {
158  return false;
159  }
160  }
161 
173  public static XryNamespace fromDisplayName(String xryNamespace) {
174  String normalizedNamespace = xryNamespace.trim().toLowerCase();
175  for (XryNamespace keyChoice : XryNamespace.values()) {
176  if (normalizedNamespace.equals(keyChoice.name)) {
177  return keyChoice;
178  }
179  }
180 
181  throw new IllegalArgumentException(String.format("Namespace [%s] was not found."
182  + " All namespaces should be tested with contains.", xryNamespace));
183  }
184  }
185 
189  private enum XryMetaKey {
190  REFERENCE_NUMBER("reference number"),
191  SEGMENT_COUNT("segments"),
192  SEGMENT_NUMBER("segment number");
193 
194  private final String name;
195 
196  XryMetaKey(String name) {
197  this.name = name;
198  }
199 
200  public String getDisplayName() {
201  return name;
202  }
203 
210  public static boolean contains(String name) {
211  try {
213  return true;
214  } catch (IllegalArgumentException ex) {
215  return false;
216  }
217  }
218 
229  public static XryMetaKey fromDisplayName(String name) {
230  String normalizedName = name.trim().toLowerCase();
231  for (XryMetaKey keyChoice : XryMetaKey.values()) {
232  if (normalizedName.equals(keyChoice.name)) {
233  return keyChoice;
234  }
235  }
236 
237  throw new IllegalArgumentException(String.format("Key [ %s ] was not found."
238  + " All keys should be tested with contains.", name));
239  }
240  }
241 
261  @Override
262  public void parse(XRYFileReader reader, Content parent, SleuthkitCase currentCase) throws IOException, TskCoreException, BlackboardException {
263  Path reportPath = reader.getReportPath();
264  logger.log(Level.INFO, String.format("[XRY DSP] Processing report at"
265  + " [ %s ]", reportPath.toString()));
266 
267  //Keep track of the reference numbers that have been parsed.
268  Set<Integer> referenceNumbersSeen = new HashSet<>();
269 
270  while (reader.hasNextEntity()) {
271  String xryEntity = reader.nextEntity();
272 
273  // This call will combine all segmented text into a single key value pair
274  List<XRYKeyValuePair> pairs = getXRYKeyValuePairs(xryEntity, reader, referenceNumbersSeen);
275 
276  // Transform all the data from XRY land into the appropriate CommHelper
277  // data types.
278  final String messageType = PARSER_NAME;
279  CommunicationDirection direction = CommunicationDirection.UNKNOWN;
280  String senderId = null;
281  final List<String> recipientIdsList = new ArrayList<>();
282  long dateTime = 0L;
283  MessageReadStatus readStatus = MessageReadStatus.UNKNOWN;
284  final String subject = null;
285  String text = null;
286  final String threadId = null;
287  final Collection<BlackboardAttribute> otherAttributes = new ArrayList<>();
288 
289  for(XRYKeyValuePair pair : pairs) {
290  XryNamespace namespace = XryNamespace.NONE;
291  if (XryNamespace.contains(pair.getNamespace())) {
292  namespace = XryNamespace.fromDisplayName(pair.getNamespace());
293  }
294  XryKey key = XryKey.fromDisplayName(pair.getKey());
295  String normalizedValue = pair.getValue().toLowerCase().trim();
296 
297  switch (key) {
298  case TEL:
299  case NUMBER:
300  if(!XRYUtils.isPhoneValid(pair.getValue())) {
301  continue;
302  }
303 
304  // Apply namespace or direction
305  if(namespace == XryNamespace.FROM || direction == CommunicationDirection.INCOMING) {
306  senderId = pair.getValue();
307  } else if(namespace == XryNamespace.TO || direction == CommunicationDirection.OUTGOING) {
308  recipientIdsList.add(pair.getValue());
309  } else {
310  currentCase.getCommunicationsManager().createAccountFileInstance(
311  Account.Type.PHONE, pair.getValue(), PARSER_NAME, parent);
312  otherAttributes.add(new BlackboardAttribute(
313  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
314  PARSER_NAME, pair.getValue()));
315  }
316  break;
317  // Although confusing, as these are also 'name spaces', it appears
318  // later versions of XRY just made these standardized lines.
319  case FROM:
320  if(!XRYUtils.isPhoneValid(pair.getValue())) {
321  continue;
322  }
323 
324  senderId = pair.getValue();
325  break;
326  case TO:
327  if(!XRYUtils.isPhoneValid(pair.getValue())) {
328  continue;
329  }
330 
331  recipientIdsList.add(pair.getValue());
332  break;
333  case TIME:
334  try {
335  //Tranform value to seconds since epoch
336  long dateTimeSinceInEpoch = XRYUtils.calculateSecondsSinceEpoch(pair.getValue());
337  dateTime = dateTimeSinceInEpoch;
338  } catch (DateTimeParseException ex) {
339  logger.log(Level.WARNING, String.format("[%s] Assumption"
340  + " about the date time formatting of messages is "
341  + "not right. Here is the pair [ %s ]", PARSER_NAME, pair), ex);
342  }
343  break;
344  case TYPE:
345  switch (normalizedValue) {
346  case "incoming":
347  direction = CommunicationDirection.INCOMING;
348  break;
349  case "outgoing":
350  direction = CommunicationDirection.OUTGOING;
351  break;
352  case "deliver":
353  case "submit":
354  case "status report":
355  //Ignore for now.
356  break;
357  default:
358  logger.log(Level.WARNING, String.format("[%s] Unrecognized "
359  + " value for key pair [ %s ].", PARSER_NAME, pair));
360  }
361  break;
362  case STATUS:
363  switch (normalizedValue) {
364  case "read":
365  readStatus = MessageReadStatus.READ;
366  break;
367  case "unread":
368  readStatus = MessageReadStatus.UNREAD;
369  break;
370  case "deleted":
371  otherAttributes.add(new BlackboardAttribute(
372  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED,
373  PARSER_NAME, pair.getValue()));
374  break;
375  case "sending failed":
376  case "unsent":
377  case "sent":
378  //Ignoring for now.
379  break;
380  default:
381  logger.log(Level.WARNING, String.format("[%s] Unrecognized "
382  + " value for key pair [ %s ].", PARSER_NAME, pair));
383  }
384  break;
385  case TEXT:
386  case MESSAGE:
387  text = pair.getValue();
388  break;
389  case DIRECTION:
390  switch (normalizedValue) {
391  case "incoming":
392  direction = CommunicationDirection.INCOMING;
393  break;
394  case "outgoing":
395  direction = CommunicationDirection.OUTGOING;
396  break;
397  default:
398  direction = CommunicationDirection.UNKNOWN;
399  break;
400  }
401  break;
402  case SERVICE_CENTER:
403  if(!XRYUtils.isPhoneValid(pair.getValue())) {
404  continue;
405  }
406 
407  otherAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
408  PARSER_NAME, pair.getValue()));
409  break;
410  default:
411  //Otherwise, the XryKey enum contains the correct BlackboardAttribute
412  //type.
413  if (key.getType() != null) {
414  otherAttributes.add(new BlackboardAttribute(key.getType(),
415  PARSER_NAME, pair.getValue()));
416  } else {
417  logger.log(Level.INFO, String.format("[%s] Key value pair "
418  + "(in brackets) [ %s ] was recognized but "
419  + "more data or time is needed to finish implementation. Discarding... ",
420  PARSER_NAME, pair));
421  }
422  }
423  }
424 
425  CommunicationArtifactsHelper helper = new CommunicationArtifactsHelper(
426  currentCase, PARSER_NAME, parent, Account.Type.PHONE);
427 
428  helper.addMessage(messageType, direction, senderId, recipientIdsList,
429  dateTime, readStatus, subject, text, threadId, otherAttributes);
430  }
431  }
432 
437  private List<XRYKeyValuePair> getXRYKeyValuePairs(String xryEntity,
438  XRYFileReader reader, Set<Integer> referenceValues) throws IOException {
439  String[] xryLines = xryEntity.split("\n");
440  //First line of the entity is the title, each XRY entity is non-empty.
441  logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0]));
442 
443  List<XRYKeyValuePair> pairs = new ArrayList<>();
444 
445  //Count the key value pairs in the XRY entity.
446  int keyCount = getCountOfKeyValuePairs(xryLines);
447  for (int i = 1; i <= keyCount; i++) {
448  //Get the ith key value pair in the entity. Always expect to have
449  //a valid value.
450  XRYKeyValuePair pair = getKeyValuePairByIndex(xryLines, i).get();
451  if (XryMetaKey.contains(pair.getKey())) {
452  //Skip meta keys, they are being handled seperately.
453  continue;
454  }
455 
456  if (!XryKey.contains(pair.getKey())) {
457  logger.log(Level.WARNING, String.format("[XRY DSP] The following key, "
458  + "value pair (in brackets) [ %s ], "
459  + "was not recognized. Discarding...", pair));
460  continue;
461  }
462 
463  if (pair.getValue().isEmpty()) {
464  logger.log(Level.WARNING, String.format("[XRY DSP] The following key "
465  + "(in brackets) [ %s ] was recognized, but the value "
466  + "was empty. Discarding...", pair.getKey()));
467  continue;
468  }
469 
470  //Assume text and message are the only fields that can be segmented
471  //among multiple XRY entities.
472  if (pair.hasKey(XryKey.TEXT.getDisplayName())
473  || pair.hasKey(XryKey.MESSAGE.getDisplayName())) {
474  String segmentedText = getSegmentedText(xryLines, reader, referenceValues);
475  pair = new XRYKeyValuePair(pair.getKey(),
476  //Assume text is segmented by word.
477  pair.getValue() + " " + segmentedText,
478  pair.getNamespace());
479  }
480 
481  pairs.add(pair);
482  }
483 
484  return pairs;
485  }
486 
491  private Integer getCountOfKeyValuePairs(String[] xryEntity) {
492  int count = 0;
493  for (int i = 1; i < xryEntity.length; i++) {
494  if (XRYKeyValuePair.isPair(xryEntity[i])) {
495  count++;
496  }
497  }
498  return count;
499  }
500 
511  private String getSegmentedText(String[] xryEntity, XRYFileReader reader,
512  Set<Integer> referenceNumbersSeen) throws IOException {
513  Optional<Integer> referenceNumber = getMetaKeyValue(xryEntity, XryMetaKey.REFERENCE_NUMBER);
514  //Check if there is any segmented text.
515  if (!referenceNumber.isPresent()) {
516  return "";
517  }
518 
519  logger.log(Level.INFO, String.format("[XRY DSP] Message entity "
520  + "appears to be segmented with reference number [ %d ]", referenceNumber.get()));
521 
522  if (referenceNumbersSeen.contains(referenceNumber.get())) {
523  logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already "
524  + "been seen. This means that the segments are not "
525  + "contiguous. Any segments contiguous with this "
526  + "one will be aggregated and another "
527  + "(otherwise duplicate) artifact will be created.", referenceNumber.get()));
528  }
529 
530  referenceNumbersSeen.add(referenceNumber.get());
531 
532  Optional<Integer> segmentNumber = getMetaKeyValue(xryEntity, XryMetaKey.SEGMENT_NUMBER);
533  if (!segmentNumber.isPresent()) {
534  logger.log(Level.SEVERE, String.format("No segment "
535  + "number was found on the message entity"
536  + "with reference number [%d]", referenceNumber.get()));
537  return "";
538  }
539 
540  StringBuilder segmentedText = new StringBuilder();
541 
542  int currentSegmentNumber = segmentNumber.get();
543  while (reader.hasNextEntity()) {
544  //Peek at the next to see if it has the same reference number.
545  String nextEntity = reader.peek();
546  String[] nextEntityLines = nextEntity.split("\n");
547  Optional<Integer> nextReferenceNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.REFERENCE_NUMBER);
548 
549  if (!nextReferenceNumber.isPresent()
550  || !Objects.equals(nextReferenceNumber, referenceNumber)) {
551  //Don't consume the next entity. It is not related
552  //to the current message thread.
553  break;
554  }
555 
556  //Consume the entity, it is a part of the message thread.
557  reader.nextEntity();
558 
559  Optional<Integer> nextSegmentNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.SEGMENT_NUMBER);
560 
561  logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ] "
562  + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber.get()));
563 
564  if (!nextSegmentNumber.isPresent()) {
565  logger.log(Level.SEVERE, String.format("[XRY DSP] Segment with reference"
566  + " number [ %d ] did not have a segment number associated with it."
567  + " It cannot be determined if the reconstructed text will be in order.", referenceNumber.get()));
568  } else if (nextSegmentNumber.get() != currentSegmentNumber + 1) {
569  logger.log(Level.SEVERE, String.format("[XRY DSP] Contiguous "
570  + "segments are not ascending incrementally. Encountered "
571  + "segment [ %d ] after segment [ %d ]. This means the reconstructed "
572  + "text will be out of order.", nextSegmentNumber.get(), currentSegmentNumber));
573  }
574 
575  int keyCount = getCountOfKeyValuePairs(nextEntityLines);
576  for (int i = 1; i <= keyCount; i++) {
577  XRYKeyValuePair pair = getKeyValuePairByIndex(nextEntityLines, i).get();
578  if (pair.hasKey(XryKey.TEXT.getDisplayName())
579  || pair.hasKey(XryKey.MESSAGE.getDisplayName())) {
580  segmentedText.append(pair.getValue()).append(' ');
581  }
582  }
583 
584  if (nextSegmentNumber.isPresent()) {
585  currentSegmentNumber = nextSegmentNumber.get();
586  }
587  }
588 
589  //Remove the trailing space.
590  if (segmentedText.length() > 0) {
591  segmentedText.setLength(segmentedText.length() - 1);
592  }
593 
594  return segmentedText.toString();
595  }
596 
604  private Optional<Integer> getMetaKeyValue(String[] xryLines, XryMetaKey metaKey) {
605  for (String xryLine : xryLines) {
606  if (!XRYKeyValuePair.isPair(xryLine)) {
607  continue;
608  }
609 
610  XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine);
611  if (pair.hasKey(metaKey.getDisplayName())) {
612  try {
613  return Optional.of(Integer.parseInt(pair.getValue()));
614  } catch (NumberFormatException ex) {
615  logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for "
616  + "meta key [ %s ] was not an integer.", pair.getValue(), metaKey), ex);
617  }
618  }
619  }
620  return Optional.empty();
621  }
622 
632  private Optional<XRYKeyValuePair> getKeyValuePairByIndex(String[] xryLines, int index) {
633  int pairsParsed = 0;
634  String namespace = "";
635  for (int i = 1; i < xryLines.length; i++) {
636  String xryLine = xryLines[i];
637  if (XryNamespace.contains(xryLine)) {
638  namespace = xryLine.trim();
639  continue;
640  }
641 
642  if (!XRYKeyValuePair.isPair(xryLine)) {
643  logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value "
644  + "pair on this line (in brackets) [ %s ], but one was not detected."
645  + " Discarding...", xryLine));
646  continue;
647  }
648 
649  XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine);
650  String value = pair.getValue();
651  //Build up multiple lines.
652  for (; (i + 1) < xryLines.length
653  && !XRYKeyValuePair.isPair(xryLines[i + 1])
654  && !XryNamespace.contains(xryLines[i + 1]); i++) {
655  String continuedValue = xryLines[i + 1].trim();
656  //Assume multi lined values are split by word.
657  value = value + " " + continuedValue;
658  }
659 
660  pair = new XRYKeyValuePair(pair.getKey(), value, namespace);
661  pairsParsed++;
662  if (pairsParsed == index) {
663  return Optional.of(pair);
664  }
665  }
666 
667  return Optional.empty();
668  }
669 }

Copyright © 2012-2020 Basis Technology. Generated on: Wed Apr 8 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.