Autopsy  4.14.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
PlasoIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2018-2019 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.plaso;
20 
21 import java.io.BufferedReader;
22 import java.io.BufferedWriter;
23 import java.io.File;
24 import java.io.FileNotFoundException;
25 import java.io.IOException;
26 import java.io.InputStreamReader;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.nio.file.Paths;
30 import java.sql.ResultSet;
31 import java.sql.SQLException;
32 import java.text.SimpleDateFormat;
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.List;
36 import java.util.Locale;
37 import static java.util.Objects.nonNull;
38 import java.util.concurrent.TimeUnit;
39 import java.util.logging.Level;
40 import java.util.stream.Collectors;
41 import org.openide.modules.InstalledFileLocator;
42 import org.openide.util.Cancellable;
43 import org.openide.util.NbBundle;
57 import org.sleuthkit.datamodel.AbstractFile;
58 import org.sleuthkit.datamodel.Blackboard;
59 import org.sleuthkit.datamodel.Blackboard.BlackboardException;
60 import org.sleuthkit.datamodel.BlackboardArtifact;
61 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_TL_EVENT;
62 import org.sleuthkit.datamodel.BlackboardAttribute;
63 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME;
64 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION;
65 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TL_EVENT_TYPE;
66 import org.sleuthkit.datamodel.Content;
67 import org.sleuthkit.datamodel.Image;
68 import org.sleuthkit.datamodel.TskCoreException;
69 import org.sleuthkit.datamodel.TimelineEventType;
70 
74 public class PlasoIngestModule implements DataSourceIngestModule {
75 
76  private static final Logger logger = Logger.getLogger(PlasoIngestModule.class.getName());
77  private static final String MODULE_NAME = PlasoModuleFactory.getModuleName();
78 
79  private static final String PLASO = "plaso"; //NON-NLS
80  private static final String PLASO64 = "plaso-20180818-amd64";//NON-NLS
81  private static final String PLASO32 = "plaso-20180818-win32";//NON-NLS
82  private static final String LOG2TIMELINE_EXECUTABLE = "Log2timeline.exe";//NON-NLS
83  private static final String PSORT_EXECUTABLE = "psort.exe";//NON-NLS
84  private static final String COOKIE = "cookie";//NON-NLS
85  private static final int LOG2TIMELINE_WORKERS = 2;
86  private static final long TERMINATION_CHECK_INTERVAL = 5;
87  private static final TimeUnit TERMINATION_CHECK_INTERVAL_UNITS = TimeUnit.SECONDS;
88 
89  private File log2TimeLineExecutable;
90  private File psortExecutable;
91 
94  private Case currentCase;
96 
97  private Image image;
98  private AbstractFile previousFile = null; // cache used when looking up files in Autopsy DB
99 
101  this.settings = settings;
102  }
103 
104  @NbBundle.Messages({
105  "PlasoIngestModule.executable.not.found=Plaso Executable Not Found.",
106  "PlasoIngestModule.requires.windows=Plaso module requires windows."})
107  @Override
108  public void startUp(IngestJobContext context) throws IngestModuleException {
109  this.context = context;
110 
111  if (false == PlatformUtil.isWindowsOS()) {
112  throw new IngestModuleException(Bundle.PlasoIngestModule_requires_windows());
113  }
114 
115  try {
116  log2TimeLineExecutable = locateExecutable(LOG2TIMELINE_EXECUTABLE);
117  psortExecutable = locateExecutable(PSORT_EXECUTABLE);
118  } catch (FileNotFoundException exception) {
119  logger.log(Level.WARNING, "Plaso executable not found.", exception); //NON-NLS
120  throw new IngestModuleException(Bundle.PlasoIngestModule_executable_not_found(), exception);
121  }
122 
123  }
124 
125  @NbBundle.Messages({
126  "PlasoIngestModule.error.running.log2timeline=Error running log2timeline, see log file.",
127  "PlasoIngestModule.error.running.psort=Error running Psort, see log file.",
128  "PlasoIngestModule.error.creating.output.dir=Error creating Plaso module output directory.",
129  "PlasoIngestModule.starting.log2timeline=Starting Log2timeline",
130  "PlasoIngestModule.running.psort=Running Psort",
131  "PlasoIngestModule.log2timeline.cancelled=Log2timeline run was canceled",
132  "PlasoIngestModule.psort.cancelled=psort run was canceled",
133  "PlasoIngestModule.bad.imageFile=Cannot find image file name and path",
134  "PlasoIngestModule.completed=Plaso Processing Completed",
135  "PlasoIngestModule.has.run=Plaso",
136  "PlasoIngestModule.psort.fail=Plaso returned an error when sorting events. Results are not complete.",
137  "PlasoIngestModule.dataSource.not.an.image=Skipping non-disk image datasource"})
138  @Override
139  public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
140 
141  if (!(dataSource instanceof Image)) {
143  Bundle.PlasoIngestModule_has_run(),
144  Bundle.PlasoIngestModule_dataSource_not_an_image());
146  return ProcessResult.OK;
147  } else {
148  image = (Image) dataSource;
149 
150  statusHelper.switchToDeterminate(100);
151  currentCase = Case.getCurrentCase();
152  fileManager = currentCase.getServices().getFileManager();
153 
154  String currentTime = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss z", Locale.US).format(System.currentTimeMillis());//NON-NLS
155  Path moduleOutputPath = Paths.get(currentCase.getModuleDirectory(), PLASO, currentTime);
156  try {
157  Files.createDirectories(moduleOutputPath);
158  } catch (IOException ex) {
159  logger.log(Level.SEVERE, "Error creating Plaso module output directory.", ex); //NON-NLS
160  return ProcessResult.ERROR;
161  }
162 
163  // Run log2timeline
164  logger.log(Level.INFO, "Starting Plaso Run.");//NON-NLS
165  statusHelper.progress(Bundle.PlasoIngestModule_starting_log2timeline(), 0);
166  ProcessBuilder log2TimeLineCommand = buildLog2TimeLineCommand(moduleOutputPath, image);
167  try {
168  Process log2TimeLineProcess = log2TimeLineCommand.start();
169  try (BufferedReader log2TimeLineOutpout = new BufferedReader(new InputStreamReader(log2TimeLineProcess.getInputStream()))) {
170  L2TStatusProcessor statusReader = new L2TStatusProcessor(log2TimeLineOutpout, statusHelper, moduleOutputPath);
171  new Thread(statusReader, "log2timeline status reader").start(); //NON-NLS
172  ExecUtil.waitForTermination(LOG2TIMELINE_EXECUTABLE, log2TimeLineProcess, TERMINATION_CHECK_INTERVAL, TERMINATION_CHECK_INTERVAL_UNITS, new DataSourceIngestModuleProcessTerminator(context));
173  statusReader.cancel();
174  }
175 
176  if (context.dataSourceIngestIsCancelled()) {
177  logger.log(Level.INFO, "Log2timeline run was canceled"); //NON-NLS
178  return ProcessResult.OK;
179  }
180  if (Files.notExists(moduleOutputPath.resolve(PLASO))) {
181  logger.log(Level.WARNING, "Error running log2timeline: there was no storage file."); //NON-NLS
182  return ProcessResult.ERROR;
183  }
184 
185  // sort the output
186  statusHelper.progress(Bundle.PlasoIngestModule_running_psort(), 33);
187  ProcessBuilder psortCommand = buildPsortCommand(moduleOutputPath);
188  int result = ExecUtil.execute(psortCommand, new DataSourceIngestModuleProcessTerminator(context));
189  if (result != 0) {
190  logger.log(Level.SEVERE, String.format("Error running Psort, error code returned %d", result)); //NON-NLS
191  MessageNotifyUtil.Notify.error(MODULE_NAME, Bundle.PlasoIngestModule_psort_fail());
192  return ProcessResult.ERROR;
193  }
194 
195  if (context.dataSourceIngestIsCancelled()) {
196  logger.log(Level.INFO, "psort run was canceled"); //NON-NLS
197  return ProcessResult.OK;
198  }
199  Path plasoFile = moduleOutputPath.resolve("plasodb.db3"); //NON-NLS
200  if (Files.notExists(plasoFile)) {
201  logger.log(Level.SEVERE, "Error running Psort: there was no sqlite db file."); //NON-NLS
202  return ProcessResult.ERROR;
203  }
204 
205  // parse the output and make artifacts
206  createPlasoArtifacts(plasoFile.toString(), statusHelper);
207 
208  } catch (IOException ex) {
209  logger.log(Level.SEVERE, "Error running Plaso.", ex);//NON-NLS
210  return ProcessResult.ERROR;
211  }
212 
214  Bundle.PlasoIngestModule_has_run(),
215  Bundle.PlasoIngestModule_completed());
217  return ProcessResult.OK;
218  }
219  }
220 
221  private ProcessBuilder buildLog2TimeLineCommand(Path moduleOutputPath, Image image) {
222  //make a csv list of disabled parsers.
223  String parsersString = settings.getParsers().entrySet().stream()
224  .filter(entry -> entry.getValue() == false)
225  .map(entry -> "!" + entry.getKey()) // '!' prepended to parsername disables it. //NON-NLS
226  .collect(Collectors.joining(","));//NON-NLS
227 
228  ProcessBuilder processBuilder = buildProcessWithRunAsInvoker(
229  "\"" + log2TimeLineExecutable + "\"", //NON-NLS
230  "--vss-stores", "all", //NON-NLS
231  "-z", image.getTimeZone(), //NON-NLS
232  "--partitions", "all", //NON-NLS
233  "--hasher_file_size_limit", "1", //NON-NLS
234  "--hashers", "none", //NON-NLS
235  "--parsers", "\"" + parsersString + "\"",//NON-NLS
236  "--no_dependencies_check", //NON-NLS
237  "--workers", String.valueOf(LOG2TIMELINE_WORKERS),//NON-NLS
238  moduleOutputPath.resolve(PLASO).toString(),
239  image.getPaths()[0]
240  );
241  processBuilder.redirectError(moduleOutputPath.resolve("log2timeline_err.txt").toFile()); //NON-NLS
242  return processBuilder;
243  }
244 
245  static private ProcessBuilder buildProcessWithRunAsInvoker(String... commandLine) {
246  ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
247  /*
248  * Add an environment variable to force log2timeline/psort to run with
249  * the same permissions Autopsy uses.
250  */
251  processBuilder.environment().put("__COMPAT_LAYER", "RunAsInvoker"); //NON-NLS
252  return processBuilder;
253  }
254 
255  private ProcessBuilder buildPsortCommand(Path moduleOutputPath) {
256  ProcessBuilder processBuilder = buildProcessWithRunAsInvoker(
257  "\"" + psortExecutable + "\"", //NON-NLS
258  "-o", "4n6time_sqlite", //NON-NLS
259  "-w", moduleOutputPath.resolve("plasodb.db3").toString(), //NON-NLS
260  moduleOutputPath.resolve(PLASO).toString()
261  );
262 
263  processBuilder.redirectOutput(moduleOutputPath.resolve("psort_output.txt").toFile()); //NON-NLS
264  processBuilder.redirectError(moduleOutputPath.resolve("psort_err.txt").toFile()); //NON-NLS
265  return processBuilder;
266  }
267 
268  private static File locateExecutable(String executableName) throws FileNotFoundException {
269  String architectureFolder = PlatformUtil.is64BitOS() ? PLASO64 : PLASO32;
270  String executableToFindName = Paths.get(PLASO, architectureFolder, executableName).toString();
271 
272  File exeFile = InstalledFileLocator.getDefault().locate(executableToFindName, PlasoIngestModule.class.getPackage().getName(), false);
273  if (null == exeFile || exeFile.canExecute() == false) {
274  throw new FileNotFoundException(executableName + " executable not found.");
275  }
276  return exeFile;
277  }
278 
279  @NbBundle.Messages({
280  "PlasoIngestModule.exception.posting.artifact=Exception Posting artifact.",
281  "PlasoIngestModule.event.datetime=Event Date Time",
282  "PlasoIngestModule.event.description=Event Description",
283  "PlasoIngestModule.create.artifacts.cancelled=Cancelled Plaso Artifact Creation ",
284  "# {0} - file that events are from",
285  "PlasoIngestModule.artifact.progress=Adding events to case: {0}",
286  "PlasoIngestModule.info.empty.database=Plaso database was empty.",})
287  private void createPlasoArtifacts(String plasoDb, DataSourceIngestModuleProgress statusHelper) {
288  Blackboard blackboard = currentCase.getSleuthkitCase().getBlackboard();
289 
290  String sqlStatement = "SELECT substr(filename,1) AS filename, "
291  + " strftime('%s', datetime) AS epoch_date, "
292  + " description, "
293  + " source, "
294  + " type, "
295  + " sourcetype "
296  + " FROM log2timeline "
297  + " WHERE source NOT IN ('FILE', "
298  + " 'WEBHIST') " // bad dates and duplicates with what we have.
299  + " AND sourcetype NOT IN ('UNKNOWN', "
300  + " 'PE Import Time');"; // lots of bad dates //NON-NLS
301 
302  try (SQLiteDBConnect tempdbconnect = new SQLiteDBConnect("org.sqlite.JDBC", "jdbc:sqlite:" + plasoDb); //NON-NLS
303  ResultSet resultSet = tempdbconnect.executeQry(sqlStatement)) {
304 
305  boolean dbHasData = false;
306 
307  while (resultSet.next()) {
308  dbHasData = true;
309 
310  if (context.dataSourceIngestIsCancelled()) {
311  logger.log(Level.INFO, "Cancelled Plaso Artifact Creation."); //NON-NLS
312  return;
313  }
314 
315  String currentFileName = resultSet.getString("filename"); //NON-NLS
316  statusHelper.progress(Bundle.PlasoIngestModule_artifact_progress(currentFileName), 66);
317  Content resolvedFile = getAbstractFile(currentFileName);
318  if (resolvedFile == null) {
319  logger.log(Level.INFO, "File {0} from Plaso output not found in case. Associating it with the data source instead.", currentFileName);//NON-NLS
320  resolvedFile = image;
321  }
322 
323  String description = resultSet.getString("description");
324  TimelineEventType eventType = findEventSubtype(currentFileName, resultSet);
325 
326  // If the description is empty use the event type display name
327  // as the description.
328  if (description == null || description.isEmpty()) {
329  if (eventType != TimelineEventType.OTHER) {
330  description = eventType.getDisplayName();
331  } else {
332  continue;
333  }
334  }
335 
336  Collection<BlackboardAttribute> bbattributes = Arrays.asList(
337  new BlackboardAttribute(
338  TSK_DATETIME, MODULE_NAME,
339  resultSet.getLong("epoch_date")), //NON-NLS
340  new BlackboardAttribute(
341  TSK_DESCRIPTION, MODULE_NAME,
342  description),//NON-NLS
343  new BlackboardAttribute(
344  TSK_TL_EVENT_TYPE, MODULE_NAME,
345  eventType.getTypeID()));
346 
347  try {
348  BlackboardArtifact bbart = resolvedFile.newArtifact(TSK_TL_EVENT);
349  bbart.addAttributes(bbattributes);
350  try {
351  /*
352  * Post the artifact which will index the artifact for
353  * keyword search, and fire an event to notify UI of
354  * this new artifact
355  */
356  blackboard.postArtifact(bbart, MODULE_NAME);
357  } catch (BlackboardException ex) {
358  logger.log(Level.SEVERE, "Error Posting Artifact.", ex);//NON-NLS
359  }
360  } catch (TskCoreException ex) {
361  logger.log(Level.SEVERE, "Exception Adding Artifact.", ex);//NON-NLS
362  }
363  }
364 
365  // Check if there is data the db
366  if (!dbHasData) {
367  logger.log(Level.INFO, String.format("PlasoDB was empty: %s", plasoDb));
368  MessageNotifyUtil.Notify.info(MODULE_NAME, Bundle.PlasoIngestModule_info_empty_database());
369  }
370  } catch (SQLException ex) {
371  logger.log(Level.SEVERE, "Error while trying to read into a sqlite db.", ex);//NON-NLS
372  }
373  }
374 
375  private AbstractFile getAbstractFile(String file) {
376 
377  Path path = Paths.get(file);
378  String fileName = path.getFileName().toString();
379  String filePath = path.getParent().toString().replaceAll("\\\\", "/");//NON-NLS
380  if (filePath.endsWith("/") == false) {//NON-NLS
381  filePath += "/";//NON-NLS
382  }
383 
384  // check the cached file
385  //TODO: would we reduce 'cache misses' if we retrieved the events sorted by file? Is that overhead worth it?
386  if (previousFile != null
387  && previousFile.getName().equalsIgnoreCase(fileName)
388  && previousFile.getParentPath().equalsIgnoreCase(filePath)) {
389  return previousFile;
390 
391  }
392  try {
393  List<AbstractFile> abstractFiles = fileManager.findFiles(fileName, filePath);
394  if (abstractFiles.size() == 1) {// TODO: why do we bother with this check. also we don't cache the file...
395  return abstractFiles.get(0);
396  }
397  for (AbstractFile resolvedFile : abstractFiles) {
398  // double check its an exact match
399  if (filePath.equalsIgnoreCase(resolvedFile.getParentPath())) {
400  // cache it for next time
401  previousFile = resolvedFile;
402  return resolvedFile;
403  }
404  }
405  } catch (TskCoreException ex) {
406  logger.log(Level.SEVERE, "Exception finding file.", ex);
407  }
408  return null;
409  }
410 
422  private TimelineEventType findEventSubtype(String fileName, ResultSet row) throws SQLException {
423  switch (row.getString("source")) {
424  case "WEBHIST": //These shouldn't actually be present, but keeping the logic just in case...
425  if (fileName.toLowerCase().contains(COOKIE)
426  || row.getString("type").toLowerCase().contains(COOKIE)) {//NON-NLS
427 
428  return TimelineEventType.WEB_COOKIE;
429  } else {
430  return TimelineEventType.WEB_HISTORY;
431  }
432  case "EVT":
433  case "LOG":
434  return TimelineEventType.LOG_ENTRY;
435  case "REG":
436  switch (row.getString("sourcetype").toLowerCase()) {//NON-NLS
437  case "unknown : usb entries":
438  case "unknown : usbstor entries":
439  return TimelineEventType.DEVICES_ATTACHED;
440  default:
441  return TimelineEventType.REGISTRY;
442  }
443  default:
444  return TimelineEventType.OTHER;
445  }
446  }
447 
453  private static class L2TStatusProcessor implements Runnable, Cancellable {
454 
455  private final BufferedReader log2TimeLineOutpout;
457  volatile private boolean cancelled = false;
458  private final Path outputPath;
459 
460  private L2TStatusProcessor(BufferedReader log2TimeLineOutpout, DataSourceIngestModuleProgress statusHelper, Path outputPath) throws IOException {
461  this.log2TimeLineOutpout = log2TimeLineOutpout;
462  this.statusHelper = statusHelper;
463  this.outputPath = outputPath;
464  }
465 
466  @Override
467  public void run() {
468  try (BufferedWriter writer = Files.newBufferedWriter(outputPath.resolve("log2timeline_output.txt"));) {//NON-NLS
469  String line = log2TimeLineOutpout.readLine();
470  while (cancelled == false && nonNull(line)) {
471  statusHelper.progress(line);
472  writer.write(line);
473  writer.newLine();
474  line = log2TimeLineOutpout.readLine();
475  }
476  writer.flush();
477  } catch (IOException ex) {
478  logger.log(Level.WARNING, "Error reading log2timeline output stream.", ex);//NON-NLS
479  }
480  }
481 
482  @Override
483  public boolean cancel() {
484  cancelled = true;
485  return true;
486  }
487  }
488 }
static int execute(ProcessBuilder processBuilder)
Definition: ExecUtil.java:132
L2TStatusProcessor(BufferedReader log2TimeLineOutpout, DataSourceIngestModuleProgress statusHelper, Path outputPath)
static ProcessBuilder buildProcessWithRunAsInvoker(String...commandLine)
static int waitForTermination(String processName, Process process, long terminationCheckInterval, TimeUnit units, ProcessTerminator terminator)
Definition: ExecUtil.java:202
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static void info(String title, String message)
ProcessBuilder buildPsortCommand(Path moduleOutputPath)
void postMessage(final IngestMessage message)
static void error(String title, String message)
synchronized List< AbstractFile > findFiles(String fileName)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper)
ProcessBuilder buildLog2TimeLineCommand(Path moduleOutputPath, Image image)
TimelineEventType findEventSubtype(String fileName, ResultSet row)
void createPlasoArtifacts(String plasoDb, DataSourceIngestModuleProgress statusHelper)
static synchronized IngestServices getInstance()

Copyright © 2012-2020 Basis Technology. Generated on: Wed Apr 8 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.