Autopsy  4.16.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
ILeappFileProcessor.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2020 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.ileappanalyzer;
20 
21 import java.io.BufferedReader;
22 import java.io.File;
23 import java.io.FileNotFoundException;
24 import java.io.FileReader;
25 import java.io.IOException;
26 import java.io.UncheckedIOException;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.text.ParseException;
30 import java.text.SimpleDateFormat;
31 import java.util.List;
32 import java.util.ArrayList;
33 import java.util.Collection;
34 import java.util.Date;
35 import java.util.HashMap;
36 import static java.util.Locale.US;
37 import java.util.Map;
38 import java.util.logging.Level;
39 import java.util.stream.Collectors;
40 import java.util.stream.Stream;
41 import javax.xml.parsers.DocumentBuilder;
42 import javax.xml.parsers.DocumentBuilderFactory;
43 import javax.xml.parsers.ParserConfigurationException;
44 import org.apache.commons.io.FilenameUtils;
45 import org.openide.util.NbBundle;
51 import org.sleuthkit.datamodel.AbstractFile;
52 import org.sleuthkit.datamodel.Blackboard;
53 import org.sleuthkit.datamodel.BlackboardArtifact;
54 import org.sleuthkit.datamodel.BlackboardAttribute;
55 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
56 import org.sleuthkit.datamodel.Content;
57 import org.sleuthkit.datamodel.TskCoreException;
58 import org.sleuthkit.datamodel.TskException;
59 import org.w3c.dom.Document;
60 import org.w3c.dom.NamedNodeMap;
61 import org.w3c.dom.NodeList;
62 import org.xml.sax.SAXException;
63 
67 public final class ILeappFileProcessor {
68 
69  private static final Logger logger = Logger.getLogger(ILeappFileProcessor.class.getName());
70  private static final String MODULE_NAME = ILeappAnalyzerModuleFactory.getModuleName();
71 
72  private static final String XMLFILE = "ileap-artifact-attribute-reference.xml"; //NON-NLS
73 
74  private final Map<String, String> tsvFiles;
75  private final Map<String, String> tsvFileArtifacts;
76  private final Map<String, String> tsvFileArtifactComments;
77  private final Map<String, List<List<String>>> tsvFileAttributes;
78 
79  public ILeappFileProcessor() throws IOException, IngestModuleException {
80  this.tsvFiles = new HashMap<>();
81  this.tsvFileArtifacts = new HashMap<>();
82  this.tsvFileArtifactComments = new HashMap<>();
83  this.tsvFileAttributes = new HashMap<>();
84 
87 
88  }
89 
90  @NbBundle.Messages({
91  "ILeappFileProcessor.error.running.iLeapp=Error running iLeapp, see log file.",
92  "ILeappFileProcessor.error.creating.output.dir=Error creating iLeapp module output directory.",
93  "ILeappFileProcessor.starting.iLeapp=Starting iLeapp",
94  "ILeappFileProcessor.running.iLeapp=Running iLeapp",
95  "ILeappFileProcessor.has.run=iLeapp",
96  "ILeappFileProcessor.iLeapp.cancelled=iLeapp run was canceled",
97  "ILeappFileProcessor.completed=iLeapp Processing Completed",
98  "ILeappFileProcessor.error.reading.iLeapp.directory=Error reading iLeapp Output Directory"})
99 
100  public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile iLeappFile) {
101 
102  try {
103  List<String> iLeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
104  processiLeappFiles(iLeappTsvOutputFiles, iLeappFile);
105  } catch (IOException | IngestModuleException ex) {
106  logger.log(Level.SEVERE, String.format("Error trying to process iLeapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
107  return ProcessResult.ERROR;
108  }
109 
110  return ProcessResult.OK;
111  }
112 
117  private List<String> findTsvFiles(Path iLeappOutputDir) throws IngestModuleException {
118  List<String> allTsvFiles = new ArrayList<>();
119  List<String> foundTsvFiles = new ArrayList<>();
120 
121  try (Stream<Path> walk = Files.walk(iLeappOutputDir)) {
122 
123  allTsvFiles = walk.map(x -> x.toString())
124  .filter(f -> f.toLowerCase().endsWith(".tsv")).collect(Collectors.toList());
125 
126  for (String tsvFile : allTsvFiles) {
127  if (tsvFiles.containsKey(FilenameUtils.getName(tsvFile))) {
128  foundTsvFiles.add(tsvFile);
129  }
130  }
131 
132  } catch (IOException | UncheckedIOException e) {
133  throw new IngestModuleException(Bundle.ILeappFileProcessor_error_reading_iLeapp_directory() + iLeappOutputDir.toString(), e);
134  }
135 
136  return foundTsvFiles;
137 
138  }
139 
150  private void processiLeappFiles(List<String> iLeappFilesToProcess, AbstractFile iLeappImageFile) throws FileNotFoundException, IOException, IngestModuleException {
151  List<BlackboardArtifact> bbartifacts = new ArrayList<>();
152 
153  for (String iLeappFileName : iLeappFilesToProcess) {
154  String fileName = FilenameUtils.getName(iLeappFileName);
155  File iLeappFile = new File(iLeappFileName);
156  if (tsvFileAttributes.containsKey(fileName)) {
157  List<List<String>> attrList = tsvFileAttributes.get(fileName);
158  try {
159  BlackboardArtifact.Type artifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(tsvFileArtifacts.get(fileName));
160 
161  processFile(iLeappFile, attrList, fileName, artifactType, bbartifacts, iLeappImageFile);
162 
163  } catch (TskCoreException ex) {
164  // check this
165  throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex);
166  }
167  }
168 
169  }
170 
171  if (!bbartifacts.isEmpty()) {
172  postArtifacts(bbartifacts);
173  }
174 
175  }
176 
177  private void processFile(File iLeappFile, List<List<String>> attrList, String fileName, BlackboardArtifact.Type artifactType,
178  List<BlackboardArtifact> bbartifacts, AbstractFile iLeappImageFile) throws FileNotFoundException, IOException, IngestModuleException {
179  try (BufferedReader reader = new BufferedReader(new FileReader(iLeappFile))) {
180  String line = reader.readLine();
181  // Check first line, if it is null then no heading so nothing to match to, close and go to next file.
182  if (line != null) {
183  Map<Integer, String> columnNumberToProcess = findColumnsToProcess(line, attrList);
184  line = reader.readLine();
185  while (line != null) {
186  Collection<BlackboardAttribute> bbattributes = processReadLine(line, columnNumberToProcess, fileName);
187  if (!bbattributes.isEmpty()) {
188  BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), iLeappImageFile, bbattributes);
189  if (bbartifact != null) {
190  bbartifacts.add(bbartifact);
191  }
192  }
193  line = reader.readLine();
194  }
195  }
196  }
197 
198  }
199 
209  private Collection<BlackboardAttribute> processReadLine(String line, Map<Integer, String> columnNumberToProcess, String fileName) throws IngestModuleException {
210  String[] columnValues = line.split("\\t");
211 
212  Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
213 
214  for (Map.Entry<Integer, String> columnToProcess : columnNumberToProcess.entrySet()) {
215  Integer columnNumber = columnToProcess.getKey();
216  String attributeName = columnToProcess.getValue();
217 
218  try {
219  BlackboardAttribute.Type attributeType = Case.getCurrentCase().getSleuthkitCase().getAttributeType(attributeName.toUpperCase());
220  if (attributeType == null) {
221  break;
222  }
223  String attrType = attributeType.getValueType().getLabel().toUpperCase();
224  checkAttributeType(bbattributes, attrType, columnValues, columnNumber, attributeType, fileName);
225  } catch (TskCoreException ex) {
226  throw new IngestModuleException(String.format("Error getting Attribute type for Attribute Name %s", attributeName), ex); //NON-NLS
227  }
228  }
229 
230  if (tsvFileArtifactComments.containsKey(fileName)) {
231  bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, tsvFileArtifactComments.get(fileName)));
232  }
233 
234  return bbattributes;
235 
236  }
237 
238  private void checkAttributeType(Collection<BlackboardAttribute> bbattributes, String attrType, String[] columnValues, Integer columnNumber, BlackboardAttribute.Type attributeType,
239  String fileName) {
240  if (attrType.matches("STRING")) {
241  bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValues[columnNumber]));
242  } else if (attrType.matches("INTEGER")) {
243  bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Integer.valueOf(columnValues[columnNumber])));
244  } else if (attrType.matches("LONG")) {
245  bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Long.valueOf(columnValues[columnNumber])));
246  } else if (attrType.matches("DOUBLE")) {
247  bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Double.valueOf(columnValues[columnNumber])));
248  } else if (attrType.matches("BYTE")) {
249  bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Byte.valueOf(columnValues[columnNumber])));
250  } else if (attrType.matches("DATETIME")) {
251  // format of data should be the same in all the data and the format is 2020-03-28 01:00:17
252  SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-d HH:mm:ss", US);
253  Long dateLong = Long.valueOf(0);
254  try {
255  Date newDate = dateFormat.parse(columnValues[columnNumber]);
256  dateLong = newDate.getTime() / 1000;
257  bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, dateLong));
258  } catch (ParseException ex) {
259  // catching error and displaying date that could not be parsed
260  // we set the timestamp to 0 and continue on processing
261  logger.log(Level.WARNING, String.format("Failed to parse date/time %s for attribute type %s in file %s.", columnValues[columnNumber], attributeType.getDisplayName(), fileName)); //NON-NLS
262  }
263  } else if (attrType.matches("JSON")) {
264 
265  bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValues[columnNumber]));
266  } else {
267  // Log this and continue on with processing
268  logger.log(Level.WARNING, String.format("Attribute Type %s not defined.", attrType)); //NON-NLS
269  }
270 
271  }
272 
284  private Map<Integer, String> findColumnsToProcess(String line, List<List<String>> attrList) {
285  String[] columnNames = line.split("\\t");
286  HashMap<Integer, String> columnsToProcess = new HashMap<>();
287 
288  Integer columnPosition = 0;
289  for (String columnName : columnNames) {
290  // for some reason the first column of the line has unprintable characters so removing them
291  String cleanColumnName = columnName.replaceAll("[^\\n\\r\\t\\p{Print}]", "");
292  for (List<String> atList : attrList) {
293  if (atList.contains(cleanColumnName.toLowerCase())) {
294  columnsToProcess.put(columnPosition, atList.get(0));
295  break;
296  }
297  }
298  columnPosition++;
299  }
300 
301  return columnsToProcess;
302  }
303 
304  @NbBundle.Messages({
305  "ILeappFileProcessor.cannot.load.artifact.xml=Cannor load xml artifact file.",
306  "ILeappFileProcessor.cannotBuildXmlParser=Cannot buld an XML parser.",
307  "ILeappFileProcessor_cannotParseXml=Cannot Parse XML file.",
308  "ILeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact",
309  "ILeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts."
310  })
311 
315  private void loadConfigFile() throws IngestModuleException {
316  Document xmlinput;
317  try {
318  String path = PlatformUtil.getUserConfigDirectory() + File.separator + XMLFILE;
319  File f = new File(path);
320  DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
321  DocumentBuilder db = dbf.newDocumentBuilder();
322  xmlinput = db.parse(f);
323 
324  } catch (IOException e) {
325  throw new IngestModuleException(Bundle.ILeappFileProcessor_cannot_load_artifact_xml() + e.getLocalizedMessage(), e); //NON-NLS
326  } catch (ParserConfigurationException pce) {
327  throw new IngestModuleException(Bundle.ILeappFileProcessor_cannotBuildXmlParser() + pce.getLocalizedMessage(), pce); //NON-NLS
328  } catch (SAXException sxe) {
329  throw new IngestModuleException(Bundle.ILeappFileProcessor_cannotParseXml() + sxe.getLocalizedMessage(), sxe); //NON-NLS
330  }
331 
332  getFileNode(xmlinput);
333  getArtifactNode(xmlinput);
334  getAttributeNodes(xmlinput);
335 
336  }
337 
338  private void getFileNode(Document xmlinput) {
339 
340  NodeList nlist = xmlinput.getElementsByTagName("FileName"); //NON-NLS
341 
342  for (int i = 0; i < nlist.getLength(); i++) {
343  NamedNodeMap nnm = nlist.item(i).getAttributes();
344  tsvFiles.put(nnm.getNamedItem("filename").getNodeValue(), nnm.getNamedItem("description").getNodeValue());
345 
346  }
347 
348  }
349 
350  private void getArtifactNode(Document xmlinput) {
351 
352  NodeList artifactNlist = xmlinput.getElementsByTagName("ArtifactName"); //NON-NLS
353  for (int k = 0; k < artifactNlist.getLength(); k++) {
354  NamedNodeMap nnm = artifactNlist.item(k).getAttributes();
355  String artifactName = nnm.getNamedItem("artifactname").getNodeValue();
356  String comment = nnm.getNamedItem("comment").getNodeValue();
357  String parentName = artifactNlist.item(k).getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
358 
359  tsvFileArtifacts.put(parentName, artifactName);
360 
361  if (!comment.toLowerCase().matches("null")) {
362  tsvFileArtifactComments.put(parentName, comment);
363  }
364  }
365 
366  }
367 
368  private void getAttributeNodes(Document xmlinput) {
369 
370  NodeList attributeNlist = xmlinput.getElementsByTagName("AttributeName"); //NON-NLS
371  for (int k = 0; k < attributeNlist.getLength(); k++) {
372  List<String> attributeList = new ArrayList<>();
373  NamedNodeMap nnm = attributeNlist.item(k).getAttributes();
374  String attributeName = nnm.getNamedItem("attributename").getNodeValue();
375  if (!attributeName.toLowerCase().matches("null")) {
376  String columnName = nnm.getNamedItem("columnName").getNodeValue();
377  String required = nnm.getNamedItem("required").getNodeValue();
378  String parentName = attributeNlist.item(k).getParentNode().getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
379 
380  attributeList.add(attributeName.toLowerCase());
381  attributeList.add(columnName.toLowerCase());
382  attributeList.add(required.toLowerCase());
383 
384  if (tsvFileAttributes.containsKey(parentName)) {
385  List<List<String>> attrList = tsvFileAttributes.get(parentName);
386  attrList.add(attributeList);
387  tsvFileAttributes.replace(parentName, attrList);
388  } else {
389  List<List<String>> attrList = new ArrayList<>();
390  attrList.add(attributeList);
391  tsvFileAttributes.put(parentName, attrList);
392  }
393  }
394 
395  }
396  }
410  private BlackboardArtifact createArtifactWithAttributes(int type, AbstractFile abstractFile, Collection<BlackboardAttribute> bbattributes) {
411  try {
412  BlackboardArtifact bbart = abstractFile.newArtifact(type);
413  bbart.addAttributes(bbattributes);
414  return bbart;
415  } catch (TskException ex) {
416  logger.log(Level.WARNING, Bundle.ILeappFileProcessor_error_creating_new_artifacts(), ex); //NON-NLS
417  }
418  return null;
419  }
420 
427  void postArtifacts(Collection<BlackboardArtifact> artifacts) {
428  if (artifacts == null || artifacts.isEmpty()) {
429  return;
430  }
431 
432  try {
433  Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifacts(artifacts, MODULE_NAME);
434  } catch (Blackboard.BlackboardException ex) {
435  logger.log(Level.SEVERE, Bundle.ILeappFileProcessor_postartifacts_error(), ex); //NON-NLS
436  }
437  }
438 
444  private void configExtractor() throws IOException {
446  }
447 
448 }
void processFile(File iLeappFile, List< List< String >> attrList, String fileName, BlackboardArtifact.Type artifactType, List< BlackboardArtifact > bbartifacts, AbstractFile iLeappImageFile)
final Map< String, List< List< String > > > tsvFileAttributes
ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile iLeappFile)
static< T > boolean extractResourceToUserConfigDir(final Class< T > resourceClass, final String resourceFileName, boolean overWrite)
Map< Integer, String > findColumnsToProcess(String line, List< List< String >> attrList)
Collection< BlackboardAttribute > processReadLine(String line, Map< Integer, String > columnNumberToProcess, String fileName)
void processiLeappFiles(List< String > iLeappFilesToProcess, AbstractFile iLeappImageFile)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
void checkAttributeType(Collection< BlackboardAttribute > bbattributes, String attrType, String[] columnValues, Integer columnNumber, BlackboardAttribute.Type attributeType, String fileName)
BlackboardArtifact createArtifactWithAttributes(int type, AbstractFile abstractFile, Collection< BlackboardAttribute > bbattributes)

Copyright © 2012-2020 Basis Technology. Generated on: Tue Sep 22 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.