Autopsy  4.17.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
XRYMessagesFileParser.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2019-2020 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.datasourceprocessors.xry;
20 
21 import java.io.IOException;
22 import java.nio.file.Path;
23 import java.time.format.DateTimeParseException;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.HashSet;
27 import java.util.List;
28 import java.util.Objects;
29 import java.util.Optional;
30 import java.util.Set;
31 import java.util.logging.Level;
33 import org.sleuthkit.datamodel.Account;
34 import org.sleuthkit.datamodel.Blackboard.BlackboardException;
35 import org.sleuthkit.datamodel.BlackboardAttribute;
36 import org.sleuthkit.datamodel.Content;
37 import org.sleuthkit.datamodel.InvalidAccountIDException;
38 import org.sleuthkit.datamodel.SleuthkitCase;
39 import org.sleuthkit.datamodel.TskCoreException;
40 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
41 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.CommunicationDirection;
42 import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper.MessageReadStatus;
43 
47 final class XRYMessagesFileParser implements XRYFileParser {
48 
49  private static final Logger logger = Logger.getLogger(
50  XRYMessagesFileParser.class.getName());
51 
52  private static final String PARSER_NAME = "XRY DSP";
53 
58  private enum XryKey {
59  DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED),
60  DIRECTION("direction", null),
61  MESSAGE("message", null),
62  NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON),
63  TEXT("text", null),
64  TIME("time", null),
65  SERVICE_CENTER("service center", null),
66  FROM("from", null),
67  TO("to", null),
68  //The following keys either need special processing or more time and data to find a type.
69  STORAGE("storage", null),
70  NUMBER("number", null),
71  TYPE("type", null),
72  TEL("tel", null),
73  FOLDER("folder", null),
74  NAME("name", null),
75  INDEX("index", null),
76  STATUS("status", null);
77 
78  private final String name;
79  private final BlackboardAttribute.ATTRIBUTE_TYPE type;
80 
81  XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) {
82  this.name = name;
83  this.type = type;
84  }
85 
86  public BlackboardAttribute.ATTRIBUTE_TYPE getType() {
87  return type;
88  }
89 
90  public String getDisplayName() {
91  return name;
92  }
93 
100  public static boolean contains(String name) {
101  try {
102  XryKey.fromDisplayName(name);
103  return true;
104  } catch (IllegalArgumentException ex) {
105  return false;
106  }
107  }
108 
119  public static XryKey fromDisplayName(String name) {
120  String normalizedName = name.trim().toLowerCase();
121  for (XryKey keyChoice : XryKey.values()) {
122  if (normalizedName.equals(keyChoice.name)) {
123  return keyChoice;
124  }
125  }
126 
127  throw new IllegalArgumentException(String.format("Key [ %s ] was not found."
128  + " All keys should be tested with contains.", name));
129  }
130  }
131 
135  private enum XryNamespace {
136  FROM("from"),
137  PARTICIPANT("participant"),
138  TO("to"),
139  NONE(null);
140 
141  private final String name;
142 
143  XryNamespace(String name) {
144  this.name = name;
145  }
146 
154  public static boolean contains(String xryNamespace) {
155  try {
156  XryNamespace.fromDisplayName(xryNamespace);
157  return true;
158  } catch (IllegalArgumentException ex) {
159  return false;
160  }
161  }
162 
174  public static XryNamespace fromDisplayName(String xryNamespace) {
175  String normalizedNamespace = xryNamespace.trim().toLowerCase();
176  for (XryNamespace keyChoice : XryNamespace.values()) {
177  if (normalizedNamespace.equals(keyChoice.name)) {
178  return keyChoice;
179  }
180  }
181 
182  throw new IllegalArgumentException(String.format("Namespace [%s] was not found."
183  + " All namespaces should be tested with contains.", xryNamespace));
184  }
185  }
186 
190  private enum XryMetaKey {
191  REFERENCE_NUMBER("reference number"),
192  SEGMENT_COUNT("segments"),
193  SEGMENT_NUMBER("segment number");
194 
195  private final String name;
196 
197  XryMetaKey(String name) {
198  this.name = name;
199  }
200 
201  public String getDisplayName() {
202  return name;
203  }
204 
211  public static boolean contains(String name) {
212  try {
214  return true;
215  } catch (IllegalArgumentException ex) {
216  return false;
217  }
218  }
219 
230  public static XryMetaKey fromDisplayName(String name) {
231  String normalizedName = name.trim().toLowerCase();
232  for (XryMetaKey keyChoice : XryMetaKey.values()) {
233  if (normalizedName.equals(keyChoice.name)) {
234  return keyChoice;
235  }
236  }
237 
238  throw new IllegalArgumentException(String.format("Key [ %s ] was not found."
239  + " All keys should be tested with contains.", name));
240  }
241  }
242 
262  @Override
263  public void parse(XRYFileReader reader, Content parent, SleuthkitCase currentCase) throws IOException, TskCoreException, BlackboardException {
264  Path reportPath = reader.getReportPath();
265  logger.log(Level.INFO, String.format("[XRY DSP] Processing report at"
266  + " [ %s ]", reportPath.toString()));
267 
268  //Keep track of the reference numbers that have been parsed.
269  Set<Integer> referenceNumbersSeen = new HashSet<>();
270 
271  while (reader.hasNextEntity()) {
272  String xryEntity = reader.nextEntity();
273 
274  // This call will combine all segmented text into a single key value pair
275  List<XRYKeyValuePair> pairs = getXRYKeyValuePairs(xryEntity, reader, referenceNumbersSeen);
276 
277  // Transform all the data from XRY land into the appropriate CommHelper
278  // data types.
279  final String messageType = PARSER_NAME;
280  CommunicationDirection direction = CommunicationDirection.UNKNOWN;
281  String senderId = null;
282  final List<String> recipientIdsList = new ArrayList<>();
283  long dateTime = 0L;
284  MessageReadStatus readStatus = MessageReadStatus.UNKNOWN;
285  final String subject = null;
286  String text = null;
287  final String threadId = null;
288  final Collection<BlackboardAttribute> otherAttributes = new ArrayList<>();
289 
290  for(XRYKeyValuePair pair : pairs) {
291  XryNamespace namespace = XryNamespace.NONE;
292  if (XryNamespace.contains(pair.getNamespace())) {
293  namespace = XryNamespace.fromDisplayName(pair.getNamespace());
294  }
295  XryKey key = XryKey.fromDisplayName(pair.getKey());
296  String normalizedValue = pair.getValue().toLowerCase().trim();
297 
298  switch (key) {
299  case TEL:
300  case NUMBER:
301  if(!XRYUtils.isPhoneValid(pair.getValue())) {
302  continue;
303  }
304 
305  // Apply namespace or direction
306  if(namespace == XryNamespace.FROM || direction == CommunicationDirection.INCOMING) {
307  senderId = pair.getValue();
308  } else if(namespace == XryNamespace.TO || direction == CommunicationDirection.OUTGOING) {
309  recipientIdsList.add(pair.getValue());
310  } else {
311  try {
312  currentCase.getCommunicationsManager().createAccountFileInstance(
313  Account.Type.PHONE, pair.getValue(), PARSER_NAME, parent);
314  } catch (InvalidAccountIDException ex) {
315  logger.log(Level.WARNING, String.format("Invalid account identifier %s", pair.getValue()), ex);
316  }
317 
318  otherAttributes.add(new BlackboardAttribute(
319  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
320  PARSER_NAME, pair.getValue()));
321  }
322  break;
323  // Although confusing, as these are also 'name spaces', it appears
324  // later versions of XRY just made these standardized lines.
325  case FROM:
326  if(!XRYUtils.isPhoneValid(pair.getValue())) {
327  continue;
328  }
329 
330  senderId = pair.getValue();
331  break;
332  case TO:
333  if(!XRYUtils.isPhoneValid(pair.getValue())) {
334  continue;
335  }
336 
337  recipientIdsList.add(pair.getValue());
338  break;
339  case TIME:
340  try {
341  //Tranform value to seconds since epoch
342  long dateTimeSinceInEpoch = XRYUtils.calculateSecondsSinceEpoch(pair.getValue());
343  dateTime = dateTimeSinceInEpoch;
344  } catch (DateTimeParseException ex) {
345  logger.log(Level.WARNING, String.format("[%s] Assumption"
346  + " about the date time formatting of messages is "
347  + "not right. Here is the pair [ %s ]", PARSER_NAME, pair), ex);
348  }
349  break;
350  case TYPE:
351  switch (normalizedValue) {
352  case "incoming":
353  direction = CommunicationDirection.INCOMING;
354  break;
355  case "outgoing":
356  direction = CommunicationDirection.OUTGOING;
357  break;
358  case "deliver":
359  case "submit":
360  case "status report":
361  //Ignore for now.
362  break;
363  default:
364  logger.log(Level.WARNING, String.format("[%s] Unrecognized "
365  + " value for key pair [ %s ].", PARSER_NAME, pair));
366  }
367  break;
368  case STATUS:
369  switch (normalizedValue) {
370  case "read":
371  readStatus = MessageReadStatus.READ;
372  break;
373  case "unread":
374  readStatus = MessageReadStatus.UNREAD;
375  break;
376  case "deleted":
377  otherAttributes.add(new BlackboardAttribute(
378  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED,
379  PARSER_NAME, pair.getValue()));
380  break;
381  case "sending failed":
382  case "unsent":
383  case "sent":
384  //Ignoring for now.
385  break;
386  default:
387  logger.log(Level.WARNING, String.format("[%s] Unrecognized "
388  + " value for key pair [ %s ].", PARSER_NAME, pair));
389  }
390  break;
391  case TEXT:
392  case MESSAGE:
393  text = pair.getValue();
394  break;
395  case DIRECTION:
396  switch (normalizedValue) {
397  case "incoming":
398  direction = CommunicationDirection.INCOMING;
399  break;
400  case "outgoing":
401  direction = CommunicationDirection.OUTGOING;
402  break;
403  default:
404  direction = CommunicationDirection.UNKNOWN;
405  break;
406  }
407  break;
408  case SERVICE_CENTER:
409  if(!XRYUtils.isPhoneValid(pair.getValue())) {
410  continue;
411  }
412 
413  otherAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
414  PARSER_NAME, pair.getValue()));
415  break;
416  default:
417  //Otherwise, the XryKey enum contains the correct BlackboardAttribute
418  //type.
419  if (key.getType() != null) {
420  otherAttributes.add(new BlackboardAttribute(key.getType(),
421  PARSER_NAME, pair.getValue()));
422  } else {
423  logger.log(Level.INFO, String.format("[%s] Key value pair "
424  + "(in brackets) [ %s ] was recognized but "
425  + "more data or time is needed to finish implementation. Discarding... ",
426  PARSER_NAME, pair));
427  }
428  }
429  }
430 
431  CommunicationArtifactsHelper helper = new CommunicationArtifactsHelper(
432  currentCase, PARSER_NAME, parent, Account.Type.PHONE);
433 
434  helper.addMessage(messageType, direction, senderId, recipientIdsList,
435  dateTime, readStatus, subject, text, threadId, otherAttributes);
436  }
437  }
438 
443  private List<XRYKeyValuePair> getXRYKeyValuePairs(String xryEntity,
444  XRYFileReader reader, Set<Integer> referenceValues) throws IOException {
445  String[] xryLines = xryEntity.split("\n");
446  //First line of the entity is the title, each XRY entity is non-empty.
447  logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0]));
448 
449  List<XRYKeyValuePair> pairs = new ArrayList<>();
450 
451  //Count the key value pairs in the XRY entity.
452  int keyCount = getCountOfKeyValuePairs(xryLines);
453  for (int i = 1; i <= keyCount; i++) {
454  //Get the ith key value pair in the entity. Always expect to have
455  //a valid value.
456  XRYKeyValuePair pair = getKeyValuePairByIndex(xryLines, i).get();
457  if (XryMetaKey.contains(pair.getKey())) {
458  //Skip meta keys, they are being handled seperately.
459  continue;
460  }
461 
462  if (!XryKey.contains(pair.getKey())) {
463  logger.log(Level.WARNING, String.format("[XRY DSP] The following key, "
464  + "value pair (in brackets) [ %s ], "
465  + "was not recognized. Discarding...", pair));
466  continue;
467  }
468 
469  if (pair.getValue().isEmpty()) {
470  logger.log(Level.WARNING, String.format("[XRY DSP] The following key "
471  + "(in brackets) [ %s ] was recognized, but the value "
472  + "was empty. Discarding...", pair.getKey()));
473  continue;
474  }
475 
476  //Assume text and message are the only fields that can be segmented
477  //among multiple XRY entities.
478  if (pair.hasKey(XryKey.TEXT.getDisplayName())
479  || pair.hasKey(XryKey.MESSAGE.getDisplayName())) {
480  String segmentedText = getSegmentedText(xryLines, reader, referenceValues);
481  pair = new XRYKeyValuePair(pair.getKey(),
482  //Assume text is segmented by word.
483  pair.getValue() + " " + segmentedText,
484  pair.getNamespace());
485  }
486 
487  pairs.add(pair);
488  }
489 
490  return pairs;
491  }
492 
497  private Integer getCountOfKeyValuePairs(String[] xryEntity) {
498  int count = 0;
499  for (int i = 1; i < xryEntity.length; i++) {
500  if (XRYKeyValuePair.isPair(xryEntity[i])) {
501  count++;
502  }
503  }
504  return count;
505  }
506 
517  private String getSegmentedText(String[] xryEntity, XRYFileReader reader,
518  Set<Integer> referenceNumbersSeen) throws IOException {
519  Optional<Integer> referenceNumber = getMetaKeyValue(xryEntity, XryMetaKey.REFERENCE_NUMBER);
520  //Check if there is any segmented text.
521  if (!referenceNumber.isPresent()) {
522  return "";
523  }
524 
525  logger.log(Level.INFO, String.format("[XRY DSP] Message entity "
526  + "appears to be segmented with reference number [ %d ]", referenceNumber.get()));
527 
528  if (referenceNumbersSeen.contains(referenceNumber.get())) {
529  logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already "
530  + "been seen. This means that the segments are not "
531  + "contiguous. Any segments contiguous with this "
532  + "one will be aggregated and another "
533  + "(otherwise duplicate) artifact will be created.", referenceNumber.get()));
534  }
535 
536  referenceNumbersSeen.add(referenceNumber.get());
537 
538  Optional<Integer> segmentNumber = getMetaKeyValue(xryEntity, XryMetaKey.SEGMENT_NUMBER);
539  if (!segmentNumber.isPresent()) {
540  logger.log(Level.SEVERE, String.format("No segment "
541  + "number was found on the message entity"
542  + "with reference number [%d]", referenceNumber.get()));
543  return "";
544  }
545 
546  StringBuilder segmentedText = new StringBuilder();
547 
548  int currentSegmentNumber = segmentNumber.get();
549  while (reader.hasNextEntity()) {
550  //Peek at the next to see if it has the same reference number.
551  String nextEntity = reader.peek();
552  String[] nextEntityLines = nextEntity.split("\n");
553  Optional<Integer> nextReferenceNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.REFERENCE_NUMBER);
554 
555  if (!nextReferenceNumber.isPresent()
556  || !Objects.equals(nextReferenceNumber, referenceNumber)) {
557  //Don't consume the next entity. It is not related
558  //to the current message thread.
559  break;
560  }
561 
562  //Consume the entity, it is a part of the message thread.
563  reader.nextEntity();
564 
565  Optional<Integer> nextSegmentNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.SEGMENT_NUMBER);
566 
567  logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ] "
568  + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber.get()));
569 
570  if (!nextSegmentNumber.isPresent()) {
571  logger.log(Level.SEVERE, String.format("[XRY DSP] Segment with reference"
572  + " number [ %d ] did not have a segment number associated with it."
573  + " It cannot be determined if the reconstructed text will be in order.", referenceNumber.get()));
574  } else if (nextSegmentNumber.get() != currentSegmentNumber + 1) {
575  logger.log(Level.SEVERE, String.format("[XRY DSP] Contiguous "
576  + "segments are not ascending incrementally. Encountered "
577  + "segment [ %d ] after segment [ %d ]. This means the reconstructed "
578  + "text will be out of order.", nextSegmentNumber.get(), currentSegmentNumber));
579  }
580 
581  int keyCount = getCountOfKeyValuePairs(nextEntityLines);
582  for (int i = 1; i <= keyCount; i++) {
583  XRYKeyValuePair pair = getKeyValuePairByIndex(nextEntityLines, i).get();
584  if (pair.hasKey(XryKey.TEXT.getDisplayName())
585  || pair.hasKey(XryKey.MESSAGE.getDisplayName())) {
586  segmentedText.append(pair.getValue()).append(' ');
587  }
588  }
589 
590  if (nextSegmentNumber.isPresent()) {
591  currentSegmentNumber = nextSegmentNumber.get();
592  }
593  }
594 
595  //Remove the trailing space.
596  if (segmentedText.length() > 0) {
597  segmentedText.setLength(segmentedText.length() - 1);
598  }
599 
600  return segmentedText.toString();
601  }
602 
610  private Optional<Integer> getMetaKeyValue(String[] xryLines, XryMetaKey metaKey) {
611  for (String xryLine : xryLines) {
612  if (!XRYKeyValuePair.isPair(xryLine)) {
613  continue;
614  }
615 
616  XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine);
617  if (pair.hasKey(metaKey.getDisplayName())) {
618  try {
619  return Optional.of(Integer.parseInt(pair.getValue()));
620  } catch (NumberFormatException ex) {
621  logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for "
622  + "meta key [ %s ] was not an integer.", pair.getValue(), metaKey), ex);
623  }
624  }
625  }
626  return Optional.empty();
627  }
628 
638  private Optional<XRYKeyValuePair> getKeyValuePairByIndex(String[] xryLines, int index) {
639  int pairsParsed = 0;
640  String namespace = "";
641  for (int i = 1; i < xryLines.length; i++) {
642  String xryLine = xryLines[i];
643  if (XryNamespace.contains(xryLine)) {
644  namespace = xryLine.trim();
645  continue;
646  }
647 
648  if (!XRYKeyValuePair.isPair(xryLine)) {
649  logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value "
650  + "pair on this line (in brackets) [ %s ], but one was not detected."
651  + " Discarding...", xryLine));
652  continue;
653  }
654 
655  XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine);
656  String value = pair.getValue();
657  //Build up multiple lines.
658  for (; (i + 1) < xryLines.length
659  && !XRYKeyValuePair.isPair(xryLines[i + 1])
660  && !XryNamespace.contains(xryLines[i + 1]); i++) {
661  String continuedValue = xryLines[i + 1].trim();
662  //Assume multi lined values are split by word.
663  value = value + " " + continuedValue;
664  }
665 
666  pair = new XRYKeyValuePair(pair.getKey(), value, namespace);
667  pairsParsed++;
668  if (pairsParsed == index) {
669  return Optional.of(pair);
670  }
671  }
672 
673  return Optional.empty();
674  }
675 }

Copyright © 2012-2021 Basis Technology. Generated on: Tue Jan 19 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.