Autopsy  4.19.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
PlasoIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2018-2019 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.plaso;
20 
21 import java.io.BufferedReader;
22 import java.io.BufferedWriter;
23 import java.io.File;
24 import java.io.FileNotFoundException;
25 import java.io.IOException;
26 import java.io.InputStreamReader;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.nio.file.Paths;
30 import java.sql.ResultSet;
31 import java.sql.SQLException;
32 import java.text.SimpleDateFormat;
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.List;
36 import java.util.Locale;
37 import static java.util.Objects.nonNull;
38 import java.util.concurrent.TimeUnit;
39 import java.util.logging.Level;
40 import java.util.stream.Collectors;
41 import org.openide.modules.InstalledFileLocator;
42 import org.openide.util.Cancellable;
43 import org.openide.util.NbBundle;
57 import org.sleuthkit.datamodel.AbstractFile;
58 import org.sleuthkit.datamodel.Blackboard;
59 import org.sleuthkit.datamodel.Blackboard.BlackboardException;
60 import org.sleuthkit.datamodel.BlackboardArtifact;
61 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_TL_EVENT;
62 import org.sleuthkit.datamodel.BlackboardAttribute;
63 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME;
64 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION;
65 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TL_EVENT_TYPE;
66 import org.sleuthkit.datamodel.Content;
67 import org.sleuthkit.datamodel.Image;
68 import org.sleuthkit.datamodel.TskCoreException;
69 import org.sleuthkit.datamodel.TimelineEventType;
70 
74 public class PlasoIngestModule implements DataSourceIngestModule {
75 
76  private static final Logger logger = Logger.getLogger(PlasoIngestModule.class.getName());
77  private static final String MODULE_NAME = PlasoModuleFactory.getModuleName();
78 
79  private static final String PLASO = "plaso"; //NON-NLS
80  private static final String PLASO64 = "plaso-20180818-amd64";//NON-NLS
81  private static final String PLASO32 = "plaso-20180818-win32";//NON-NLS
82  private static final String LOG2TIMELINE_EXECUTABLE = "Log2timeline.exe";//NON-NLS
83  private static final String PSORT_EXECUTABLE = "psort.exe";//NON-NLS
84  private static final String COOKIE = "cookie";//NON-NLS
85  private static final int LOG2TIMELINE_WORKERS = 2;
86  private static final long TERMINATION_CHECK_INTERVAL = 5;
87  private static final TimeUnit TERMINATION_CHECK_INTERVAL_UNITS = TimeUnit.SECONDS;
88 
89  private File log2TimeLineExecutable;
90  private File psortExecutable;
91 
94  private Case currentCase;
96 
97  private Image image;
98  private AbstractFile previousFile = null; // cache used when looking up files in Autopsy DB
99 
101  this.settings = settings;
102  }
103 
104  @NbBundle.Messages({
105  "PlasoIngestModule.executable.not.found=Plaso Executable Not Found.",
106  "PlasoIngestModule.requires.windows=Plaso module requires windows."})
107  @Override
108  public void startUp(IngestJobContext context) throws IngestModuleException {
109  this.context = context;
110 
111  if (false == PlatformUtil.isWindowsOS()) {
112  throw new IngestModuleException(Bundle.PlasoIngestModule_requires_windows());
113  }
114 
115  try {
116  log2TimeLineExecutable = locateExecutable(LOG2TIMELINE_EXECUTABLE);
117  psortExecutable = locateExecutable(PSORT_EXECUTABLE);
118  } catch (FileNotFoundException exception) {
119  logger.log(Level.WARNING, "Plaso executable not found.", exception); //NON-NLS
120  throw new IngestModuleException(Bundle.PlasoIngestModule_executable_not_found(), exception);
121  }
122 
123  }
124 
125  @NbBundle.Messages({
126  "PlasoIngestModule.error.running.log2timeline=Error running log2timeline, see log file.",
127  "PlasoIngestModule.error.running.psort=Error running Psort, see log file.",
128  "PlasoIngestModule.error.creating.output.dir=Error creating Plaso module output directory.",
129  "PlasoIngestModule.starting.log2timeline=Starting Log2timeline",
130  "PlasoIngestModule.running.psort=Running Psort",
131  "PlasoIngestModule.log2timeline.cancelled=Log2timeline run was canceled",
132  "PlasoIngestModule.psort.cancelled=psort run was canceled",
133  "PlasoIngestModule.bad.imageFile=Cannot find image file name and path",
134  "PlasoIngestModule.completed=Plaso Processing Completed",
135  "PlasoIngestModule.has.run=Plaso",
136  "PlasoIngestModule.psort.fail=Plaso returned an error when sorting events. Results are not complete.",
137  "PlasoIngestModule.dataSource.not.an.image=Skipping non-disk image datasource"})
138  @Override
139  public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
140 
141  if (!(dataSource instanceof Image)) {
143  Bundle.PlasoIngestModule_has_run(),
144  Bundle.PlasoIngestModule_dataSource_not_an_image());
146  return ProcessResult.OK;
147  } else {
148  image = (Image) dataSource;
149 
150  statusHelper.switchToDeterminate(100);
151  currentCase = Case.getCurrentCase();
152  fileManager = currentCase.getServices().getFileManager();
153 
154  String currentTime = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss z", Locale.US).format(System.currentTimeMillis());//NON-NLS
155  Path moduleOutputPath = Paths.get(currentCase.getModuleDirectory(), PLASO, currentTime);
156  try {
157  Files.createDirectories(moduleOutputPath);
158  } catch (IOException ex) {
159  logger.log(Level.SEVERE, "Error creating Plaso module output directory.", ex); //NON-NLS
160  return ProcessResult.ERROR;
161  }
162 
163  // Run log2timeline
164  logger.log(Level.INFO, "Starting Plaso Run.");//NON-NLS
165  statusHelper.progress(Bundle.PlasoIngestModule_starting_log2timeline(), 0);
166  ProcessBuilder log2TimeLineCommand = buildLog2TimeLineCommand(moduleOutputPath, image);
167  try {
168  Process log2TimeLineProcess = log2TimeLineCommand.start();
169  try (BufferedReader log2TimeLineOutpout = new BufferedReader(new InputStreamReader(log2TimeLineProcess.getInputStream()))) {
170  L2TStatusProcessor statusReader = new L2TStatusProcessor(log2TimeLineOutpout, statusHelper, moduleOutputPath);
171  new Thread(statusReader, "log2timeline status reader").start(); //NON-NLS
172  ExecUtil.waitForTermination(LOG2TIMELINE_EXECUTABLE, log2TimeLineProcess, TERMINATION_CHECK_INTERVAL, TERMINATION_CHECK_INTERVAL_UNITS, new DataSourceIngestModuleProcessTerminator(context));
173  statusReader.cancel();
174  }
175 
176  if (context.dataSourceIngestIsCancelled()) {
177  logger.log(Level.INFO, "Log2timeline run was canceled"); //NON-NLS
178  return ProcessResult.OK;
179  }
180  if (Files.notExists(moduleOutputPath.resolve(PLASO))) {
181  logger.log(Level.WARNING, "Error running log2timeline: there was no storage file."); //NON-NLS
182  return ProcessResult.ERROR;
183  }
184 
185  // sort the output
186  statusHelper.progress(Bundle.PlasoIngestModule_running_psort(), 33);
187  ProcessBuilder psortCommand = buildPsortCommand(moduleOutputPath);
188  int result = ExecUtil.execute(psortCommand, new DataSourceIngestModuleProcessTerminator(context));
189  if (result != 0) {
190  logger.log(Level.SEVERE, String.format("Error running Psort, error code returned %d", result)); //NON-NLS
191  MessageNotifyUtil.Notify.error(MODULE_NAME, Bundle.PlasoIngestModule_psort_fail());
192  return ProcessResult.ERROR;
193  }
194 
195  if (context.dataSourceIngestIsCancelled()) {
196  logger.log(Level.INFO, "psort run was canceled"); //NON-NLS
197  return ProcessResult.OK;
198  }
199  Path plasoFile = moduleOutputPath.resolve("plasodb.db3"); //NON-NLS
200  if (Files.notExists(plasoFile)) {
201  logger.log(Level.SEVERE, "Error running Psort: there was no sqlite db file."); //NON-NLS
202  return ProcessResult.ERROR;
203  }
204 
205  // parse the output and make artifacts
206  createPlasoArtifacts(plasoFile.toString(), statusHelper);
207 
208  } catch (IOException ex) {
209  logger.log(Level.SEVERE, "Error running Plaso.", ex);//NON-NLS
210  return ProcessResult.ERROR;
211  }
212 
214  Bundle.PlasoIngestModule_has_run(),
215  Bundle.PlasoIngestModule_completed());
217  return ProcessResult.OK;
218  }
219  }
220 
221  private ProcessBuilder buildLog2TimeLineCommand(Path moduleOutputPath, Image image) {
222  //make a csv list of disabled parsers.
223  String parsersString = settings.getParsers().entrySet().stream()
224  .filter(entry -> entry.getValue() == false)
225  .map(entry -> "!" + entry.getKey()) // '!' prepended to parsername disables it. //NON-NLS
226  .collect(Collectors.joining(","));//NON-NLS
227 
228  ProcessBuilder processBuilder = buildProcessWithRunAsInvoker(
229  "\"" + log2TimeLineExecutable + "\"", //NON-NLS
230  "--vss-stores", "all", //NON-NLS
231  "-z", image.getTimeZone(), //NON-NLS
232  "--partitions", "all", //NON-NLS
233  "--hasher_file_size_limit", "1", //NON-NLS
234  "--hashers", "none", //NON-NLS
235  "--parsers", "\"" + parsersString + "\"",//NON-NLS
236  "--no_dependencies_check", //NON-NLS
237  "--workers", String.valueOf(LOG2TIMELINE_WORKERS),//NON-NLS
238  moduleOutputPath.resolve(PLASO).toString(),
239  image.getPaths()[0]
240  );
241  processBuilder.redirectError(moduleOutputPath.resolve("log2timeline_err.txt").toFile()); //NON-NLS
242  return processBuilder;
243  }
244 
245  static private ProcessBuilder buildProcessWithRunAsInvoker(String... commandLine) {
246  ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
247  /*
248  * Add an environment variable to force log2timeline/psort to run with
249  * the same permissions Autopsy uses.
250  */
251  processBuilder.environment().put("__COMPAT_LAYER", "RunAsInvoker"); //NON-NLS
252  return processBuilder;
253  }
254 
255  private ProcessBuilder buildPsortCommand(Path moduleOutputPath) {
256  ProcessBuilder processBuilder = buildProcessWithRunAsInvoker(
257  "\"" + psortExecutable + "\"", //NON-NLS
258  "-o", "4n6time_sqlite", //NON-NLS
259  "-w", moduleOutputPath.resolve("plasodb.db3").toString(), //NON-NLS
260  moduleOutputPath.resolve(PLASO).toString()
261  );
262 
263  processBuilder.redirectOutput(moduleOutputPath.resolve("psort_output.txt").toFile()); //NON-NLS
264  processBuilder.redirectError(moduleOutputPath.resolve("psort_err.txt").toFile()); //NON-NLS
265  return processBuilder;
266  }
267 
268  private static File locateExecutable(String executableName) throws FileNotFoundException {
269  String architectureFolder = PlatformUtil.is64BitOS() ? PLASO64 : PLASO32;
270  String executableToFindName = Paths.get(PLASO, architectureFolder, executableName).toString();
271 
272  File exeFile = InstalledFileLocator.getDefault().locate(executableToFindName, PlasoIngestModule.class.getPackage().getName(), false);
273  if (null == exeFile || exeFile.canExecute() == false) {
274  throw new FileNotFoundException(executableName + " executable not found.");
275  }
276  return exeFile;
277  }
278 
279  @NbBundle.Messages({
280  "PlasoIngestModule.exception.posting.artifact=Exception Posting artifact.",
281  "PlasoIngestModule.event.datetime=Event Date Time",
282  "PlasoIngestModule.event.description=Event Description",
283  "PlasoIngestModule.create.artifacts.cancelled=Cancelled Plaso Artifact Creation ",
284  "# {0} - file that events are from",
285  "PlasoIngestModule.artifact.progress=Adding events to case: {0}",
286  "PlasoIngestModule.info.empty.database=Plaso database was empty.",})
287  private void createPlasoArtifacts(String plasoDb, DataSourceIngestModuleProgress statusHelper) {
288  Blackboard blackboard = currentCase.getSleuthkitCase().getBlackboard();
289 
290  String sqlStatement = "SELECT substr(filename,1) AS filename, "
291  + " strftime('%s', datetime) AS epoch_date, "
292  + " description, "
293  + " source, "
294  + " type, "
295  + " sourcetype "
296  + " FROM log2timeline "
297  + " WHERE source NOT IN ('FILE', "
298  + " 'WEBHIST') " // bad dates and duplicates with what we have.
299  + " AND sourcetype NOT IN ('UNKNOWN', "
300  + " 'PE Import Time');"; // lots of bad dates //NON-NLS
301 
302  try (SQLiteDBConnect tempdbconnect = new SQLiteDBConnect("org.sqlite.JDBC", "jdbc:sqlite:" + plasoDb); //NON-NLS
303  ResultSet resultSet = tempdbconnect.executeQry(sqlStatement)) {
304 
305  boolean dbHasData = false;
306 
307  while (resultSet.next()) {
308  dbHasData = true;
309 
310  if (context.dataSourceIngestIsCancelled()) {
311  logger.log(Level.INFO, "Cancelled Plaso Artifact Creation."); //NON-NLS
312  return;
313  }
314 
315  String currentFileName = resultSet.getString("filename"); //NON-NLS
316  statusHelper.progress(Bundle.PlasoIngestModule_artifact_progress(currentFileName), 66);
317  Content resolvedFile = getAbstractFile(currentFileName);
318  if (resolvedFile == null) {
319  logger.log(Level.INFO, "File {0} from Plaso output not found in case. Associating it with the data source instead.", currentFileName);//NON-NLS
320  resolvedFile = image;
321  }
322 
323  String description = resultSet.getString("description");
324  TimelineEventType eventType = findEventSubtype(currentFileName, resultSet);
325 
326  // If the description is empty use the event type display name
327  // as the description.
328  if (description == null || description.isEmpty()) {
329  if (eventType != TimelineEventType.STANDARD_ARTIFACT_CATCH_ALL) {
330  description = eventType.getDisplayName();
331  } else {
332  continue;
333  }
334  }
335 
336  Collection<BlackboardAttribute> bbattributes = Arrays.asList(
337  new BlackboardAttribute(
338  TSK_DATETIME, MODULE_NAME,
339  resultSet.getLong("epoch_date")), //NON-NLS
340  new BlackboardAttribute(
341  TSK_DESCRIPTION, MODULE_NAME,
342  description),//NON-NLS
343  new BlackboardAttribute(
344  TSK_TL_EVENT_TYPE, MODULE_NAME,
345  eventType.getTypeID()));
346 
347  try {
348  BlackboardArtifact bbart = resolvedFile.newDataArtifact(new BlackboardArtifact.Type(TSK_TL_EVENT), bbattributes);
349  try {
350  /*
351  * Post the artifact which will index the artifact for
352  * keyword search, and fire an event to notify UI of
353  * this new artifact
354  */
355  blackboard.postArtifact(bbart, MODULE_NAME);
356  } catch (BlackboardException ex) {
357  logger.log(Level.SEVERE, "Error Posting Artifact.", ex);//NON-NLS
358  }
359  } catch (TskCoreException ex) {
360  logger.log(Level.SEVERE, "Exception Adding Artifact.", ex);//NON-NLS
361  }
362  }
363 
364  // Check if there is data the db
365  if (!dbHasData) {
366  logger.log(Level.INFO, String.format("PlasoDB was empty: %s", plasoDb));
367  MessageNotifyUtil.Notify.info(MODULE_NAME, Bundle.PlasoIngestModule_info_empty_database());
368  }
369  } catch (SQLException ex) {
370  logger.log(Level.SEVERE, "Error while trying to read into a sqlite db.", ex);//NON-NLS
371  }
372  }
373 
374  private AbstractFile getAbstractFile(String file) {
375 
376  Path path = Paths.get(file);
377  String fileName = path.getFileName().toString();
378  String filePath = path.getParent().toString().replaceAll("\\\\", "/");//NON-NLS
379  if (filePath.endsWith("/") == false) {//NON-NLS
380  filePath += "/";//NON-NLS
381  }
382 
383  // check the cached file
384  //TODO: would we reduce 'cache misses' if we retrieved the events sorted by file? Is that overhead worth it?
385  if (previousFile != null
386  && previousFile.getName().equalsIgnoreCase(fileName)
387  && previousFile.getParentPath().equalsIgnoreCase(filePath)) {
388  return previousFile;
389 
390  }
391  try {
392  List<AbstractFile> abstractFiles = fileManager.findFiles(fileName, filePath);
393  if (abstractFiles.size() == 1) {// TODO: why do we bother with this check. also we don't cache the file...
394  return abstractFiles.get(0);
395  }
396  for (AbstractFile resolvedFile : abstractFiles) {
397  // double check its an exact match
398  if (filePath.equalsIgnoreCase(resolvedFile.getParentPath())) {
399  // cache it for next time
400  previousFile = resolvedFile;
401  return resolvedFile;
402  }
403  }
404  } catch (TskCoreException ex) {
405  logger.log(Level.SEVERE, "Exception finding file.", ex);
406  }
407  return null;
408  }
409 
421  private TimelineEventType findEventSubtype(String fileName, ResultSet row) throws SQLException {
422  switch (row.getString("source")) {
423  case "WEBHIST": //These shouldn't actually be present, but keeping the logic just in case...
424  if (fileName.toLowerCase().contains(COOKIE)
425  || row.getString("type").toLowerCase().contains(COOKIE)) {//NON-NLS
426 
427  return TimelineEventType.WEB_COOKIE;
428  } else {
429  return TimelineEventType.WEB_HISTORY;
430  }
431  case "EVT":
432  case "LOG":
433  return TimelineEventType.LOG_ENTRY;
434  case "REG":
435  switch (row.getString("sourcetype").toLowerCase()) {//NON-NLS
436  case "unknown : usb entries":
437  case "unknown : usbstor entries":
438  return TimelineEventType.DEVICES_ATTACHED;
439  default:
440  return TimelineEventType.REGISTRY;
441  }
442  default:
443  return TimelineEventType.STANDARD_ARTIFACT_CATCH_ALL;
444  }
445  }
446 
452  private static class L2TStatusProcessor implements Runnable, Cancellable {
453 
454  private final BufferedReader log2TimeLineOutpout;
456  volatile private boolean cancelled = false;
457  private final Path outputPath;
458 
459  private L2TStatusProcessor(BufferedReader log2TimeLineOutpout, DataSourceIngestModuleProgress statusHelper, Path outputPath) throws IOException {
460  this.log2TimeLineOutpout = log2TimeLineOutpout;
461  this.statusHelper = statusHelper;
462  this.outputPath = outputPath;
463  }
464 
465  @Override
466  public void run() {
467  try (BufferedWriter writer = Files.newBufferedWriter(outputPath.resolve("log2timeline_output.txt"));) {//NON-NLS
468  String line = log2TimeLineOutpout.readLine();
469  while (cancelled == false && nonNull(line)) {
470  statusHelper.progress(line);
471  writer.write(line);
472  writer.newLine();
473  line = log2TimeLineOutpout.readLine();
474  }
475  writer.flush();
476  } catch (IOException ex) {
477  logger.log(Level.WARNING, "Error reading log2timeline output stream.", ex);//NON-NLS
478  }
479  }
480 
481  @Override
482  public boolean cancel() {
483  cancelled = true;
484  return true;
485  }
486  }
487 }
static int execute(ProcessBuilder processBuilder)
Definition: ExecUtil.java:172
List< AbstractFile > findFiles(String fileName)
L2TStatusProcessor(BufferedReader log2TimeLineOutpout, DataSourceIngestModuleProgress statusHelper, Path outputPath)
static ProcessBuilder buildProcessWithRunAsInvoker(String...commandLine)
static int waitForTermination(String processName, Process process, long terminationCheckInterval, TimeUnit units, ProcessTerminator terminator)
Definition: ExecUtil.java:264
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static void info(String title, String message)
ProcessBuilder buildPsortCommand(Path moduleOutputPath)
void postMessage(final IngestMessage message)
static void error(String title, String message)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper)
ProcessBuilder buildLog2TimeLineCommand(Path moduleOutputPath, Image image)
TimelineEventType findEventSubtype(String fileName, ResultSet row)
void createPlasoArtifacts(String plasoDb, DataSourceIngestModuleProgress statusHelper)
static synchronized IngestServices getInstance()

Copyright © 2012-2021 Basis Technology. Generated on: Thu Sep 30 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.