Autopsy  4.8.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
HashDbIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.hashdatabase;
20 
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.concurrent.atomic.AtomicLong;
28 import java.util.logging.Level;
29 import org.openide.util.NbBundle;
30 import org.openide.util.NbBundle.Messages;
44 import org.sleuthkit.datamodel.AbstractFile;
45 import org.sleuthkit.datamodel.BlackboardArtifact;
46 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
47 import org.sleuthkit.datamodel.BlackboardAttribute;
48 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
49 import org.sleuthkit.datamodel.HashHitInfo;
50 import org.sleuthkit.datamodel.HashUtility;
51 import org.sleuthkit.datamodel.SleuthkitCase;
52 import org.sleuthkit.datamodel.TskCoreException;
53 import org.sleuthkit.datamodel.TskData;
54 import org.sleuthkit.datamodel.TskException;
55 
59 @Messages({
60  "HashDbIngestModule.noKnownBadHashDbSetMsg=No notable hash set.",
61  "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=Notable file search will not be executed.",
62  "HashDbIngestModule.noKnownHashDbSetMsg=No known hash set.",
63  "HashDbIngestModule.knownFileSearchWillNotExecuteWarn=Known file search will not be executed."
64 })
65 public class HashDbIngestModule implements FileIngestModule {
66 
67  private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName());
68  private static final int MAX_COMMENT_SIZE = 500;
69  private final IngestServices services = IngestServices.getInstance();
70  private final SleuthkitCase skCase;
71  private final HashDbManager hashDbManager = HashDbManager.getInstance();
72  private final HashLookupModuleSettings settings;
73  private final List<HashDb> knownBadHashSets = new ArrayList<>();
74  private final List<HashDb> knownHashSets = new ArrayList<>();
75  private long jobId;
76  private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs = new HashMap<>();
77  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
79 
83  private static class IngestJobTotals {
84 
85  private final AtomicLong totalKnownBadCount = new AtomicLong(0);
86  private final AtomicLong totalCalctime = new AtomicLong(0);
87  private final AtomicLong totalLookuptime = new AtomicLong(0);
88  }
89 
90  private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) {
91  IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId);
92  if (totals == null) {
93  totals = new HashDbIngestModule.IngestJobTotals();
94  totalsForIngestJobs.put(ingestJobId, totals);
95  }
96  return totals;
97  }
98 
108  HashDbIngestModule(HashLookupModuleSettings settings) throws NoCurrentCaseException {
109  this.settings = settings;
111  }
112 
113  @Override
115  jobId = context.getJobId();
116  if (!hashDbManager.verifyAllDatabasesLoadedCorrectly()) {
117  throw new IngestModuleException("Could not load all hash sets");
118  }
119  updateEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets);
120  updateEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets);
121 
122  if (refCounter.incrementAndGet(jobId) == 1) {
123  // initialize job totals
124  getTotalsForIngestJobs(jobId);
125 
126  // if first module for this job then post error msgs if needed
127  if (knownBadHashSets.isEmpty()) {
129  HashLookupModuleFactory.getModuleName(),
130  Bundle.HashDbIngestModule_noKnownBadHashDbSetMsg(),
131  Bundle.HashDbIngestModule_knownBadFileSearchWillNotExecuteWarn()));
132  }
133 
134  if (knownHashSets.isEmpty()) {
136  HashLookupModuleFactory.getModuleName(),
137  Bundle.HashDbIngestModule_noKnownHashDbSetMsg(),
138  Bundle.HashDbIngestModule_knownFileSearchWillNotExecuteWarn()));
139  }
140  }
141  }
142 
149  private void updateEnabledHashSets(List<HashDb> allHashSets, List<HashDb> enabledHashSets) {
150  enabledHashSets.clear();
151  for (HashDb db : allHashSets) {
152  if (settings.isHashSetEnabled(db)) {
153  try {
154  if (db.isValid()) {
155  enabledHashSets.add(db);
156  }
157  } catch (TskCoreException ex) {
158  logger.log(Level.WARNING, "Error getting index status for " + db.getDisplayName() + " hash set", ex); //NON-NLS
159  }
160  }
161  }
162  }
163 
164  @Messages({
165  "# {0} - File name",
166  "HashDbIngestModule.dialogTitle.errorFindingArtifacts=Error Finding Artifacts: {0}",
167  "# {0} - File name",
168  "HashDbIngestModule.errorMessage.lookingForFileArtifacts=Error encountered while looking for existing artifacts for {0}."
169  })
170  @Override
171  public ProcessResult process(AbstractFile file) {
172  try {
174  } catch (NoCurrentCaseException ex) {
175  logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS
176  return ProcessResult.ERROR;
177  }
178 
179  // Skip unallocated space files.
180  if ((file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
181  || file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
182  return ProcessResult.OK;
183  }
184 
185  /*
186  * Skip directories. One reason for this is because we won't accurately
187  * calculate hashes of NTFS directories that have content that spans the
188  * IDX_ROOT and IDX_ALLOC artifacts. So we disable that until a solution
189  * for it is developed.
190  */
191  if (file.isDir()) {
192  return ProcessResult.OK;
193  }
194 
195  // bail out if we have no hashes set
196  if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) {
197  return ProcessResult.OK;
198  }
199 
200  // Safely get a reference to the totalsForIngestJobs object
201  IngestJobTotals totals = getTotalsForIngestJobs(jobId);
202 
203  // calc hash value
204  String name = file.getName();
205  long fileId = file.getId();
206  String md5Hash = file.getMd5Hash();
207  if (md5Hash == null || md5Hash.isEmpty()) {
208  try {
209  TimingMetric metric = HealthMonitor.getTimingMetric("Disk Reads: Hash calculation");
210  long calcstart = System.currentTimeMillis();
211  md5Hash = HashUtility.calculateMd5Hash(file);
212  if (file.getSize() > 0) {
213  // Surprisingly, the hash calculation does not seem to be correlated that
214  // strongly with file size until the files get large.
215  // Only normalize if the file size is greater than ~1MB.
216  if (file.getSize() < 1000000) {
218  } else {
219  // In testing, this normalization gave reasonable resuls
220  HealthMonitor.submitNormalizedTimingMetric(metric, file.getSize() / 500000);
221  }
222  }
223  file.setMd5Hash(md5Hash);
224  long delta = (System.currentTimeMillis() - calcstart);
225  totals.totalCalctime.addAndGet(delta);
226 
227  } catch (IOException ex) {
228  logger.log(Level.WARNING, String.format("Error calculating hash of file '%s' (id=%d).", name, fileId), ex); //NON-NLS
230  HashLookupModuleFactory.getModuleName(),
231  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.fileReadErrorMsg", name),
232  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.calcHashValueErr", name)));
233  return ProcessResult.ERROR;
234  }
235  }
236 
237  // look up in notable first
238  boolean foundBad = false;
240  for (HashDb db : knownBadHashSets) {
241  try {
242  long lookupstart = System.currentTimeMillis();
243  HashHitInfo hashInfo = db.lookupMD5(file);
244  if (null != hashInfo) {
245  foundBad = true;
246  totals.totalKnownBadCount.incrementAndGet();
247 
248  file.setKnown(TskData.FileKnown.BAD);
249 
250  String hashSetName = db.getDisplayName();
251 
252  String comment = "";
253  ArrayList<String> comments = hashInfo.getComments();
254  int i = 0;
255  for (String c : comments) {
256  if (++i > 1) {
257  comment += " ";
258  }
259  comment += c;
260  if (comment.length() > MAX_COMMENT_SIZE) {
261  comment = comment.substring(0, MAX_COMMENT_SIZE) + "...";
262  break;
263  }
264  }
265 
266  /*
267  * We have a match. Now create an artifact if it is
268  * determined that one hasn't been created yet.
269  */
270  List<BlackboardAttribute> attributesList = new ArrayList<>();
271  attributesList.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, HashLookupModuleFactory.getModuleName(), hashSetName));
272  try {
273  org.sleuthkit.datamodel.Blackboard tskBlackboard = skCase.getBlackboard();
274  if (tskBlackboard.artifactExists(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT, attributesList) == false) {
275  postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
276  }
277  } catch (TskCoreException ex) {
278  logger.log(Level.SEVERE, String.format(
279  "A problem occurred while checking for existing artifacts for file '%s' (id=%d).", name, fileId), ex); //NON-NLS
281  HashLookupModuleFactory.getModuleName(),
282  Bundle.HashDbIngestModule_dialogTitle_errorFindingArtifacts(name),
283  Bundle.HashDbIngestModule_errorMessage_lookingForFileArtifacts(name)));
284  ret = ProcessResult.ERROR;
285  }
286  }
287  long delta = (System.currentTimeMillis() - lookupstart);
288  totals.totalLookuptime.addAndGet(delta);
289 
290  } catch (TskException ex) {
291  logger.log(Level.WARNING, String.format(
292  "Couldn't lookup notable hash for file '%s' (id=%d) - see sleuthkit log for details", name, fileId), ex); //NON-NLS
294  HashLookupModuleFactory.getModuleName(),
295  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", name),
296  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.lookingUpKnownBadHashValueErr", name)));
297  ret = ProcessResult.ERROR;
298  }
299  }
300 
301  // If the file is not in the notable sets, search for it in the known sets.
302  // Any hit is sufficient to classify it as known, and there is no need to create
303  // a hit artifact or send a message to the application inbox.
304  if (!foundBad) {
305  for (HashDb db : knownHashSets) {
306  try {
307  long lookupstart = System.currentTimeMillis();
308  if (db.lookupMD5Quick(file)) {
309  file.setKnown(TskData.FileKnown.KNOWN);
310  break;
311  }
312  long delta = (System.currentTimeMillis() - lookupstart);
313  totals.totalLookuptime.addAndGet(delta);
314 
315  } catch (TskException ex) {
316  logger.log(Level.WARNING, String.format(
317  "Couldn't lookup known hash for file '%s' (id=%d) - see sleuthkit log for details", name, fileId), ex); //NON-NLS
319  HashLookupModuleFactory.getModuleName(),
320  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", name),
321  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.lookingUpKnownHashValueErr", name)));
322  ret = ProcessResult.ERROR;
323  }
324  }
325  }
326 
327  return ret;
328  }
329 
340  @Messages({
341  "HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."
342  })
343  private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) {
344  try {
345  String moduleName = HashLookupModuleFactory.getModuleName();
346  BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT);
347  Collection<BlackboardAttribute> attributes = new ArrayList<>();
348  //TODO Revisit usage of deprecated constructor as per TSK-583
349  //BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, "Known Bad", hashSetName);
350  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, moduleName, hashSetName));
351  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, moduleName, md5Hash));
352  attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, comment));
353 
354  badFile.addAttributes(attributes);
355 
356  try {
357  // index the artifact for keyword search
358  blackboard.indexArtifact(badFile);
359  } catch (Blackboard.BlackboardException ex) {
360  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + badFile.getArtifactID(), ex); //NON-NLS
362  Bundle.HashDbIngestModule_indexError_message(), badFile.getDisplayName());
363  }
364 
365  if (showInboxMessage) {
366  StringBuilder detailsSb = new StringBuilder();
367  //details
368  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
369  //hit
370  detailsSb.append("<tr>"); //NON-NLS
371  detailsSb.append("<th>") //NON-NLS
372  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.fileName"))
373  .append("</th>"); //NON-NLS
374  detailsSb.append("<td>") //NON-NLS
375  .append(abstractFile.getName())
376  .append("</td>"); //NON-NLS
377  detailsSb.append("</tr>"); //NON-NLS
378 
379  detailsSb.append("<tr>"); //NON-NLS
380  detailsSb.append("<th>") //NON-NLS
381  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.md5Hash"))
382  .append("</th>"); //NON-NLS
383  detailsSb.append("<td>").append(md5Hash).append("</td>"); //NON-NLS
384  detailsSb.append("</tr>"); //NON-NLS
385 
386  detailsSb.append("<tr>"); //NON-NLS
387  detailsSb.append("<th>") //NON-NLS
388  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.hashsetName"))
389  .append("</th>"); //NON-NLS
390  detailsSb.append("<td>").append(hashSetName).append("</td>"); //NON-NLS
391  detailsSb.append("</tr>"); //NON-NLS
392 
393  detailsSb.append("</table>"); //NON-NLS
394 
396  NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.knownBadMsg", abstractFile.getName()),
397  detailsSb.toString(),
398  abstractFile.getName() + md5Hash,
399  badFile));
400  }
401  services.fireModuleDataEvent(new ModuleDataEvent(moduleName, ARTIFACT_TYPE.TSK_HASHSET_HIT, Collections.singletonList(badFile)));
402  } catch (TskException ex) {
403  logger.log(Level.WARNING, "Error creating blackboard artifact", ex); //NON-NLS
404  }
405  }
406 
414  private static synchronized void postSummary(long jobId,
415  List<HashDb> knownBadHashSets, List<HashDb> knownHashSets) {
416  IngestJobTotals jobTotals = getTotalsForIngestJobs(jobId);
417  totalsForIngestJobs.remove(jobId);
418 
419  if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
420  StringBuilder detailsSb = new StringBuilder();
421  //details
422  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
423 
424  detailsSb.append("<tr><td>") //NON-NLS
425  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.knownBadsFound"))
426  .append("</td>"); //NON-NLS
427  detailsSb.append("<td>").append(jobTotals.totalKnownBadCount.get()).append("</td></tr>"); //NON-NLS
428 
429  detailsSb.append("<tr><td>") //NON-NLS
430  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalCalcTime"))
431  .append("</td><td>").append(jobTotals.totalCalctime.get()).append("</td></tr>\n"); //NON-NLS
432  detailsSb.append("<tr><td>") //NON-NLS
433  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalLookupTime"))
434  .append("</td><td>").append(jobTotals.totalLookuptime.get()).append("</td></tr>\n"); //NON-NLS
435  detailsSb.append("</table>"); //NON-NLS
436 
437  detailsSb.append("<p>") //NON-NLS
438  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.databasesUsed"))
439  .append("</p>\n<ul>"); //NON-NLS
440  for (HashDb db : knownBadHashSets) {
441  detailsSb.append("<li>").append(db.getHashSetName()).append("</li>\n"); //NON-NLS
442  }
443 
444  detailsSb.append("</ul>"); //NON-NLS
445 
448  HashLookupModuleFactory.getModuleName(),
449  NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.hashLookupResults"),
450  detailsSb.toString()));
451  }
452  }
453 
454  @Override
455  public void shutDown() {
456  if (refCounter.decrementAndGet(jobId) == 0) {
457  postSummary(jobId, knownBadHashSets, knownHashSets);
458  }
459  }
460 }
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context)
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId)
void updateEnabledHashSets(List< HashDb > allHashSets, List< HashDb > enabledHashSets)
static TimingMetric getTimingMetric(String name)
void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage)
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
static void submitTimingMetric(TimingMetric metric)
static synchronized void postSummary(long jobId, List< HashDb > knownBadHashSets, List< HashDb > knownHashSets)
static void error(String title, String message)
synchronized void indexArtifact(BlackboardArtifact artifact)
Definition: Blackboard.java:58
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static IngestMessage createWarningMessage(String source, String subject, String detailsHtml)
static void submitNormalizedTimingMetric(TimingMetric metric, long normalization)
static synchronized IngestServices getInstance()

Copyright © 2012-2018 Basis Technology. Generated on: Thu Oct 4 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.