Autopsy  4.13.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
XRYMessagesFileParser.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2019 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.datasourceprocessors.xry;
20 
21 import java.io.IOException;
22 import java.nio.file.Path;
23 import java.time.Instant;
24 import java.time.LocalDateTime;
25 import java.time.OffsetDateTime;
26 import java.time.ZoneId;
27 import java.time.ZonedDateTime;
28 import java.time.format.DateTimeFormatter;
29 import java.time.format.DateTimeParseException;
30 import java.time.temporal.TemporalAccessor;
31 import java.time.temporal.TemporalQueries;
32 import java.util.ArrayList;
33 import java.util.HashSet;
34 import java.util.List;
35 import java.util.Objects;
36 import java.util.Optional;
37 import java.util.Set;
38 import java.util.logging.Level;
40 import org.sleuthkit.datamodel.BlackboardArtifact;
41 import org.sleuthkit.datamodel.BlackboardAttribute;
42 import org.sleuthkit.datamodel.Content;
43 import org.sleuthkit.datamodel.TskCoreException;
44 
48 final class XRYMessagesFileParser implements XRYFileParser {
49 
50  private static final Logger logger = Logger.getLogger(
51  XRYMessagesFileParser.class.getName());
52 
53  private static final String PARSER_NAME = "XRY DSP";
54 
55  //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch()
56  //function for more details.
57  private static final DateTimeFormatter DATE_TIME_PARSER
58  = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y");
59 
60  private static final String DEVICE_LOCALE = "(device)";
61  private static final String NETWORK_LOCALE = "(network)";
62 
63  private static final int READ = 1;
64  private static final int UNREAD = 0;
65 
70  private enum XryKey {
71  DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED),
72  DIRECTION("direction", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION),
73  MESSAGE("message", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT),
74  NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON),
75  TEXT("text", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT),
76  TIME("time", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME),
77  SERVICE_CENTER("service center", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER),
78  FROM("from", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM),
79  TO("to", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO),
80  //The following keys either need special processing or more time and data to find a type.
81  STORAGE("storage", null),
82  NUMBER("number", null),
83  TYPE("type", null),
84  TEL("tel", null),
85  FOLDER("folder", null),
86  NAME("name", null),
87  INDEX("index", null),
88  STATUS("status", null);
89 
90  private final String name;
91  private final BlackboardAttribute.ATTRIBUTE_TYPE type;
92 
93  XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) {
94  this.name = name;
95  this.type = type;
96  }
97 
98  public BlackboardAttribute.ATTRIBUTE_TYPE getType() {
99  return type;
100  }
101 
102  public String getDisplayName() {
103  return name;
104  }
105 
112  public static boolean contains(String name) {
113  try {
114  XryKey.fromDisplayName(name);
115  return true;
116  } catch (IllegalArgumentException ex) {
117  return false;
118  }
119  }
120 
131  public static XryKey fromDisplayName(String name) {
132  String normalizedName = name.trim().toLowerCase();
133  for (XryKey keyChoice : XryKey.values()) {
134  if (normalizedName.equals(keyChoice.name)) {
135  return keyChoice;
136  }
137  }
138 
139  throw new IllegalArgumentException(String.format("Key [ %s ] was not found."
140  + " All keys should be tested with contains.", name));
141  }
142  }
143 
147  private enum XryNamespace {
148  FROM("from"),
149  PARTICIPANT("participant"),
150  TO("to"),
151  NONE(null);
152 
153  private final String name;
154 
155  XryNamespace(String name) {
156  this.name = name;
157  }
158 
166  public static boolean contains(String xryNamespace) {
167  try {
168  XryNamespace.fromDisplayName(xryNamespace);
169  return true;
170  } catch (IllegalArgumentException ex) {
171  return false;
172  }
173  }
174 
186  public static XryNamespace fromDisplayName(String xryNamespace) {
187  String normalizedNamespace = xryNamespace.trim().toLowerCase();
188  for (XryNamespace keyChoice : XryNamespace.values()) {
189  if (normalizedNamespace.equals(keyChoice.name)) {
190  return keyChoice;
191  }
192  }
193 
194  throw new IllegalArgumentException(String.format("Namespace [%s] was not found."
195  + " All namespaces should be tested with contains.", xryNamespace));
196  }
197  }
198 
202  private enum XryMetaKey {
203  REFERENCE_NUMBER("reference number"),
204  SEGMENT_COUNT("segments"),
205  SEGMENT_NUMBER("segment number");
206 
207  private final String name;
208 
209  XryMetaKey(String name) {
210  this.name = name;
211  }
212 
213  public String getDisplayName() {
214  return name;
215  }
216 
223  public static boolean contains(String name) {
224  try {
226  return true;
227  } catch (IllegalArgumentException ex) {
228  return false;
229  }
230  }
231 
242  public static XryMetaKey fromDisplayName(String name) {
243  String normalizedName = name.trim().toLowerCase();
244  for (XryMetaKey keyChoice : XryMetaKey.values()) {
245  if (normalizedName.equals(keyChoice.name)) {
246  return keyChoice;
247  }
248  }
249 
250  throw new IllegalArgumentException(String.format("Key [ %s ] was not found."
251  + " All keys should be tested with contains.", name));
252  }
253  }
254 
274  @Override
275  public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException {
276  Path reportPath = reader.getReportPath();
277  logger.log(Level.INFO, String.format("[XRY DSP] Processing report at"
278  + " [ %s ]", reportPath.toString()));
279 
280  //Keep track of the reference numbers that have been parsed.
281  Set<Integer> referenceNumbersSeen = new HashSet<>();
282 
283  while (reader.hasNextEntity()) {
284  String xryEntity = reader.nextEntity();
285  List<BlackboardAttribute> attributes = getBlackboardAttributes(xryEntity, reader, referenceNumbersSeen);
286  //Only create artifacts with non-empty attributes.
287  if (!attributes.isEmpty()) {
288  BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE);
289  artifact.addAttributes(attributes);
290  }
291  }
292  }
293 
298  private List<BlackboardAttribute> getBlackboardAttributes(String xryEntity,
299  XRYFileReader reader, Set<Integer> referenceValues) throws IOException {
300  String[] xryLines = xryEntity.split("\n");
301  //First line of the entity is the title, each XRY entity is non-empty.
302  logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0]));
303 
304  List<BlackboardAttribute> attributes = new ArrayList<>();
305 
306  //Count the key value pairs in the XRY entity.
307  int keyCount = getCountOfKeyValuePairs(xryLines);
308  for (int i = 1; i <= keyCount; i++) {
309  //Get the ith key value pair in the entity. Always expect to have
310  //a valid value.
311  XRYKeyValuePair pair = getKeyValuePairByIndex(xryLines, i).get();
312  if (XryMetaKey.contains(pair.getKey())) {
313  //Skip meta keys, they are being handled seperately.
314  continue;
315  }
316 
317  if (!XryKey.contains(pair.getKey())) {
318  logger.log(Level.WARNING, String.format("[XRY DSP] The following key, "
319  + "value pair (in brackets) [ %s ], "
320  + "was not recognized. Discarding...", pair));
321  continue;
322  }
323 
324  if (pair.getValue().isEmpty()) {
325  logger.log(Level.WARNING, String.format("[XRY DSP] The following key "
326  + "(in brackets) [ %s ] was recognized, but the value "
327  + "was empty. Discarding...", pair.getKey()));
328  continue;
329  }
330 
331  //Assume text and message are the only fields that can be segmented
332  //among multiple XRY entities.
333  if (pair.hasKey(XryKey.TEXT.getDisplayName())
334  || pair.hasKey(XryKey.MESSAGE.getDisplayName())) {
335  String segmentedText = getSegmentedText(xryLines, reader, referenceValues);
336  pair = new XRYKeyValuePair(pair.getKey(),
337  //Assume text is segmented by word.
338  pair.getValue() + " " + segmentedText,
339  pair.getNamespace());
340  }
341 
342  //Get the corresponding blackboard attribute, if any.
343  Optional<BlackboardAttribute> attribute = getBlackboardAttribute(pair);
344  if (attribute.isPresent()) {
345  attributes.add(attribute.get());
346  }
347  }
348 
349  return attributes;
350  }
351 
356  private Integer getCountOfKeyValuePairs(String[] xryEntity) {
357  int count = 0;
358  for (int i = 1; i < xryEntity.length; i++) {
359  if (XRYKeyValuePair.isPair(xryEntity[i])) {
360  count++;
361  }
362  }
363  return count;
364  }
365 
376  private String getSegmentedText(String[] xryEntity, XRYFileReader reader,
377  Set<Integer> referenceNumbersSeen) throws IOException {
378  Optional<Integer> referenceNumber = getMetaKeyValue(xryEntity, XryMetaKey.REFERENCE_NUMBER);
379  //Check if there is any segmented text.
380  if (!referenceNumber.isPresent()) {
381  return "";
382  }
383 
384  logger.log(Level.INFO, String.format("[XRY DSP] Message entity "
385  + "appears to be segmented with reference number [ %d ]", referenceNumber.get()));
386 
387  if (referenceNumbersSeen.contains(referenceNumber.get())) {
388  logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already "
389  + "been seen. This means that the segments are not "
390  + "contiguous. Any segments contiguous with this "
391  + "one will be aggregated and another "
392  + "(otherwise duplicate) artifact will be created.", referenceNumber.get()));
393  }
394 
395  referenceNumbersSeen.add(referenceNumber.get());
396 
397  Optional<Integer> segmentNumber = getMetaKeyValue(xryEntity, XryMetaKey.SEGMENT_NUMBER);
398  if (!segmentNumber.isPresent()) {
399  logger.log(Level.SEVERE, String.format("No segment "
400  + "number was found on the message entity"
401  + "with reference number [%d]", referenceNumber.get()));
402  return "";
403  }
404 
405  StringBuilder segmentedText = new StringBuilder();
406 
407  int currentSegmentNumber = segmentNumber.get();
408  while (reader.hasNextEntity()) {
409  //Peek at the next to see if it has the same reference number.
410  String nextEntity = reader.peek();
411  String[] nextEntityLines = nextEntity.split("\n");
412  Optional<Integer> nextReferenceNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.REFERENCE_NUMBER);
413 
414  if (!nextReferenceNumber.isPresent()
415  || !Objects.equals(nextReferenceNumber, referenceNumber)) {
416  //Don't consume the next entity. It is not related
417  //to the current message thread.
418  break;
419  }
420 
421  //Consume the entity, it is a part of the message thread.
422  reader.nextEntity();
423 
424  Optional<Integer> nextSegmentNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.SEGMENT_NUMBER);
425 
426  logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ] "
427  + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber.get()));
428 
429  if (!nextSegmentNumber.isPresent()) {
430  logger.log(Level.SEVERE, String.format("[XRY DSP] Segment with reference"
431  + " number [ %d ] did not have a segment number associated with it."
432  + " It cannot be determined if the reconstructed text will be in order.", referenceNumber.get()));
433  } else if (nextSegmentNumber.get() != currentSegmentNumber + 1) {
434  logger.log(Level.SEVERE, String.format("[XRY DSP] Contiguous "
435  + "segments are not ascending incrementally. Encountered "
436  + "segment [ %d ] after segment [ %d ]. This means the reconstructed "
437  + "text will be out of order.", nextSegmentNumber.get(), currentSegmentNumber));
438  }
439 
440  int keyCount = getCountOfKeyValuePairs(nextEntityLines);
441  for (int i = 1; i <= keyCount; i++) {
442  XRYKeyValuePair pair = getKeyValuePairByIndex(nextEntityLines, i).get();
443  if (pair.hasKey(XryKey.TEXT.getDisplayName())
444  || pair.hasKey(XryKey.MESSAGE.getDisplayName())) {
445  segmentedText.append(pair.getValue()).append(' ');
446  }
447  }
448 
449  if (nextSegmentNumber.isPresent()) {
450  currentSegmentNumber = nextSegmentNumber.get();
451  }
452  }
453 
454  //Remove the trailing space.
455  if (segmentedText.length() > 0) {
456  segmentedText.setLength(segmentedText.length() - 1);
457  }
458 
459  return segmentedText.toString();
460  }
461 
469  private Optional<Integer> getMetaKeyValue(String[] xryLines, XryMetaKey metaKey) {
470  for (String xryLine : xryLines) {
471  if (!XRYKeyValuePair.isPair(xryLine)) {
472  continue;
473  }
474 
475  XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine);
476  if (pair.hasKey(metaKey.getDisplayName())) {
477  try {
478  return Optional.of(Integer.parseInt(pair.getValue()));
479  } catch (NumberFormatException ex) {
480  logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for "
481  + "meta key [ %s ] was not an integer.", pair.getValue(), metaKey), ex);
482  }
483  }
484  }
485  return Optional.empty();
486  }
487 
497  private Optional<XRYKeyValuePair> getKeyValuePairByIndex(String[] xryLines, int index) {
498  int pairsParsed = 0;
499  String namespace = "";
500  for (int i = 1; i < xryLines.length; i++) {
501  String xryLine = xryLines[i];
502  if (XryNamespace.contains(xryLine)) {
503  namespace = xryLine.trim();
504  continue;
505  }
506 
507  if (!XRYKeyValuePair.isPair(xryLine)) {
508  logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value "
509  + "pair on this line (in brackets) [ %s ], but one was not detected."
510  + " Discarding...", xryLine));
511  continue;
512  }
513 
514  XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine);
515  String value = pair.getValue();
516  //Build up multiple lines.
517  for (; (i + 1) < xryLines.length
518  && !XRYKeyValuePair.isPair(xryLines[i + 1])
519  && !XryNamespace.contains(xryLines[i + 1]); i++) {
520  String continuedValue = xryLines[i + 1].trim();
521  //Assume multi lined values are split by word.
522  value = value + " " + continuedValue;
523  }
524 
525  pair = new XRYKeyValuePair(pair.getKey(), value, namespace);
526  pairsParsed++;
527  if (pairsParsed == index) {
528  return Optional.of(pair);
529  }
530  }
531 
532  return Optional.empty();
533  }
534 
544  private Optional<BlackboardAttribute> getBlackboardAttribute(XRYKeyValuePair pair) {
545  XryNamespace namespace = XryNamespace.NONE;
546  if (XryNamespace.contains(pair.getNamespace())) {
547  namespace = XryNamespace.fromDisplayName(pair.getNamespace());
548  }
549  XryKey key = XryKey.fromDisplayName(pair.getKey());
550  String normalizedValue = pair.getValue().toLowerCase().trim();
551 
552  switch (key) {
553  case TEL:
554  case NUMBER:
555  switch (namespace) {
556  case FROM:
557  return Optional.of(new BlackboardAttribute(
558  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM,
559  PARSER_NAME, pair.getValue()));
560  case TO:
561  case PARTICIPANT:
562  return Optional.of(new BlackboardAttribute(
563  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO,
564  PARSER_NAME, pair.getValue()));
565  default:
566  return Optional.of(new BlackboardAttribute(
567  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
568  PARSER_NAME, pair.getValue()));
569  }
570  case TIME:
571  try {
572  //Tranform value to seconds since epoch
573  long dateTimeSinceInEpoch = calculateSecondsSinceEpoch(pair.getValue());
574  return Optional.of(new BlackboardAttribute(
575  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START,
576  PARSER_NAME, dateTimeSinceInEpoch));
577  } catch (DateTimeParseException ex) {
578  logger.log(Level.WARNING, String.format("[XRY DSP] Assumption"
579  + " about the date time formatting of messages is "
580  + "not right. Here is the pair [ %s ]", pair), ex);
581  return Optional.empty();
582  }
583  case TYPE:
584  switch (normalizedValue) {
585  case "incoming":
586  case "outgoing":
587  return Optional.of(new BlackboardAttribute(
588  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION,
589  PARSER_NAME, pair.getValue()));
590  case "deliver":
591  case "submit":
592  case "status report":
593  //Ignore for now.
594  return Optional.empty();
595  default:
596  logger.log(Level.WARNING, String.format("[XRY DSP] Unrecognized "
597  + " value for key pair [ %s ].", pair));
598  return Optional.empty();
599  }
600  case STATUS:
601  switch (normalizedValue) {
602  case "read":
603  return Optional.of(new BlackboardAttribute(
604  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS,
605  PARSER_NAME, READ));
606  case "unread":
607  return Optional.of(new BlackboardAttribute(
608  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS,
609  PARSER_NAME, UNREAD));
610  case "sending failed":
611  case "deleted":
612  case "unsent":
613  case "sent":
614  //Ignore for now.
615  return Optional.empty();
616  default:
617  logger.log(Level.WARNING, String.format("[XRY DSP] Unrecognized "
618  + " value for key pair [ %s ].", pair));
619  return Optional.empty();
620  }
621  default:
622  //Otherwise, the XryKey enum contains the correct BlackboardAttribute
623  //type.
624  if (key.getType() != null) {
625  return Optional.of(new BlackboardAttribute(key.getType(),
626  PARSER_NAME, pair.getValue()));
627  }
628 
629  logger.log(Level.INFO, String.format("[XRY DSP] Key value pair "
630  + "(in brackets) [ %s ] was recognized but "
631  + "more data or time is needed to finish implementation. Discarding... ", pair));
632 
633  return Optional.empty();
634  }
635  }
636 
645  private String removeDateTimeLocale(String dateTime) {
646  String result = dateTime;
647  int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE);
648  if (deviceIndex != -1) {
649  result = result.substring(0, deviceIndex);
650  }
651  int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE);
652  if (networkIndex != -1) {
653  result = result.substring(0, networkIndex);
654  }
655  return result;
656  }
657 
664  private long calculateSecondsSinceEpoch(String dateTime) {
665  String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim();
682  String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale);
690  String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT");
691  TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT,
692  ZonedDateTime::from,
693  LocalDateTime::from,
694  OffsetDateTime::from);
695  //Query for the ZoneID
696  if (result.query(TemporalQueries.zoneId()) == null) {
697  //If none, assumed GMT+0.
698  return ZonedDateTime.of(LocalDateTime.from(result),
699  ZoneId.of("GMT")).toEpochSecond();
700  } else {
701  return Instant.from(result).getEpochSecond();
702  }
703  }
704 
713  private String reverseOrderOfDateTimeComponents(String dateTime) {
714  StringBuilder reversedDateTime = new StringBuilder(dateTime.length());
715  String[] dateTimeComponents = dateTime.split(" ");
716  for (String component : dateTimeComponents) {
717  reversedDateTime.insert(0, " ").insert(0, component);
718  }
719  return reversedDateTime.toString().trim();
720  }
721 }

Copyright © 2012-2019 Basis Technology. Generated on: Tue Jan 7 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.