Autopsy  4.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
HashDbIngestModule.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011 - 2013 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.modules.hashdatabase;
20 
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.concurrent.atomic.AtomicLong;
27 import java.util.logging.Level;
28 import org.openide.util.NbBundle;
29 import org.openide.util.NbBundle.Messages;
51 
52 @NbBundle.Messages({
53  "HashDbIngestModule.noKnownBadHashDbSetMsg=No known bad hash database set.",
54  "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=Known bad file search will not be executed.",
55  "HashDbIngestModule.noKnownHashDbSetMsg=No known hash database set.",
56  "HashDbIngestModule.knownFileSearchWillNotExecuteWarn=Known file search will not be executed."
57 })
58 public class HashDbIngestModule implements FileIngestModule {
59 
60  private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName());
61  private static final int MAX_COMMENT_SIZE = 500;
62  private final IngestServices services = IngestServices.getInstance();
63  private final SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase();
64  private final HashDbManager hashDbManager = HashDbManager.getInstance();
65  private final HashLookupModuleSettings settings;
66  private List<HashDb> knownBadHashSets = new ArrayList<>();
67  private List<HashDb> knownHashSets = new ArrayList<>();
68  private long jobId;
69  private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs = new HashMap<>();
70  private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
72 
73  private static class IngestJobTotals {
74 
75  private AtomicLong totalKnownBadCount = new AtomicLong(0);
76  private AtomicLong totalCalctime = new AtomicLong(0);
77  private AtomicLong totalLookuptime = new AtomicLong(0);
78  }
79 
80  private static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId) {
81  IngestJobTotals totals = totalsForIngestJobs.get(ingestJobId);
82  if (totals == null) {
83  totals = new HashDbIngestModule.IngestJobTotals();
84  totalsForIngestJobs.put(ingestJobId, totals);
85  }
86  return totals;
87  }
88 
89  HashDbIngestModule(HashLookupModuleSettings settings) {
90  this.settings = settings;
91  }
92 
93  @Override
95  jobId = context.getJobId();
96  if (!hashDbManager.verifyAllDatabasesLoadedCorrectly()) {
97  throw new IngestModuleException("Could not load all hash databases");
98  }
99  updateEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets);
100  updateEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets);
101 
102  if (refCounter.incrementAndGet(jobId) == 1) {
103  // initialize job totals
104  getTotalsForIngestJobs(jobId);
105 
106  // if first module for this job then post error msgs if needed
107  if (knownBadHashSets.isEmpty()) {
109  HashLookupModuleFactory.getModuleName(),
110  Bundle.HashDbIngestModule_noKnownBadHashDbSetMsg(),
111  Bundle.HashDbIngestModule_knownBadFileSearchWillNotExecuteWarn()));
112  }
113 
114  if (knownHashSets.isEmpty()) {
116  HashLookupModuleFactory.getModuleName(),
117  Bundle.HashDbIngestModule_noKnownHashDbSetMsg(),
118  Bundle.HashDbIngestModule_knownFileSearchWillNotExecuteWarn()));
119  }
120  }
121  }
122 
129  private void updateEnabledHashSets(List<HashDb> allHashSets, List<HashDb> enabledHashSets) {
130  enabledHashSets.clear();
131  for (HashDb db : allHashSets) {
132  if (settings.isHashSetEnabled(db.getHashSetName())) {
133  try {
134  if (db.hasIndex()) {
135  enabledHashSets.add(db);
136  }
137  } catch (TskCoreException ex) {
138  logger.log(Level.WARNING, "Error getting index status for " + db.getHashSetName() + " hash database", ex); //NON-NLS
139  }
140  }
141  }
142  }
143 
144  @Override
146  blackboard = Case.getCurrentCase().getServices().getBlackboard();
147 
148  // Skip unallocated space files.
150  file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
151  return ProcessResult.OK;
152  }
153 
154  /*
155  * Skip directories. One reason for this is because we won't accurately
156  * calculate hashes of NTFS directories that have content that spans the
157  * IDX_ROOT and IDX_ALLOC artifacts. So we disable that until a solution
158  * for it is developed.
159  */
160  if (file.isDir()) {
161  return ProcessResult.OK;
162  }
163 
164  // bail out if we have no hashes set
165  if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) {
166  return ProcessResult.OK;
167  }
168 
169  // Safely get a reference to the totalsForIngestJobs object
170  IngestJobTotals totals = getTotalsForIngestJobs(jobId);
171 
172  // calc hash value
173  String name = file.getName();
174  String md5Hash = file.getMd5Hash();
175  if (md5Hash == null || md5Hash.isEmpty()) {
176  try {
177  long calcstart = System.currentTimeMillis();
178  md5Hash = HashUtility.calculateMd5(file);
179  long delta = (System.currentTimeMillis() - calcstart);
180  totals.totalCalctime.addAndGet(delta);
181 
182  } catch (IOException ex) {
183  logger.log(Level.WARNING, "Error calculating hash of file " + name, ex); //NON-NLS
185  HashLookupModuleFactory.getModuleName(),
186  NbBundle.getMessage(this.getClass(),
187  "HashDbIngestModule.fileReadErrorMsg",
188  name),
189  NbBundle.getMessage(this.getClass(),
190  "HashDbIngestModule.calcHashValueErr",
191  name)));
192  return ProcessResult.ERROR;
193  }
194  }
195 
196  // look up in known bad first
197  boolean foundBad = false;
199  for (HashDb db : knownBadHashSets) {
200  try {
201  long lookupstart = System.currentTimeMillis();
202  HashHitInfo hashInfo = db.lookupMD5(file);
203  if (null != hashInfo) {
204  foundBad = true;
205  totals.totalKnownBadCount.incrementAndGet();
206 
207  try {
208  skCase.setKnown(file, TskData.FileKnown.BAD);
209  } catch (TskException ex) {
210  logger.log(Level.WARNING, "Couldn't set known bad state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
212  HashLookupModuleFactory.getModuleName(),
213  NbBundle.getMessage(this.getClass(),
214  "HashDbIngestModule.hashLookupErrorMsg",
215  name),
216  NbBundle.getMessage(this.getClass(),
217  "HashDbIngestModule.settingKnownBadStateErr",
218  name)));
219  ret = ProcessResult.ERROR;
220  }
221  String hashSetName = db.getHashSetName();
222 
223  String comment = "";
224  ArrayList<String> comments = hashInfo.getComments();
225  int i = 0;
226  for (String c : comments) {
227  if (++i > 1) {
228  comment += " ";
229  }
230  comment += c;
231  if (comment.length() > MAX_COMMENT_SIZE) {
232  comment = comment.substring(0, MAX_COMMENT_SIZE) + "...";
233  break;
234  }
235  }
236 
237  postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
238  }
239  long delta = (System.currentTimeMillis() - lookupstart);
240  totals.totalLookuptime.addAndGet(delta);
241 
242  } catch (TskException ex) {
243  logger.log(Level.WARNING, "Couldn't lookup known bad hash for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
245  HashLookupModuleFactory.getModuleName(),
246  NbBundle.getMessage(this.getClass(),
247  "HashDbIngestModule.hashLookupErrorMsg",
248  name),
249  NbBundle.getMessage(this.getClass(),
250  "HashDbIngestModule.lookingUpKnownBadHashValueErr",
251  name)));
252  ret = ProcessResult.ERROR;
253  }
254  }
255 
256  // If the file is not in the known bad sets, search for it in the known sets.
257  // Any hit is sufficient to classify it as known, and there is no need to create
258  // a hit artifact or send a message to the application inbox.
259  if (!foundBad) {
260  for (HashDb db : knownHashSets) {
261  try {
262  long lookupstart = System.currentTimeMillis();
263  if (db.lookupMD5Quick(file)) {
264  try {
265  skCase.setKnown(file, TskData.FileKnown.KNOWN);
266  break;
267  } catch (TskException ex) {
268  logger.log(Level.WARNING, "Couldn't set known state for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
269  ret = ProcessResult.ERROR;
270  }
271  }
272  long delta = (System.currentTimeMillis() - lookupstart);
273  totals.totalLookuptime.addAndGet(delta);
274 
275  } catch (TskException ex) {
276  logger.log(Level.WARNING, "Couldn't lookup known hash for file " + name + " - see sleuthkit log for details", ex); //NON-NLS
278  HashLookupModuleFactory.getModuleName(),
279  NbBundle.getMessage(this.getClass(),
280  "HashDbIngestModule.hashLookupErrorMsg",
281  name),
282  NbBundle.getMessage(this.getClass(),
283  "HashDbIngestModule.lookingUpKnownHashValueErr",
284  name)));
285  ret = ProcessResult.ERROR;
286  }
287  }
288  }
289 
290  return ret;
291  }
292 
293  @Messages({"HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."})
294  private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) {
295  try {
296  String MODULE_NAME = NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.moduleName");
297 
299  //TODO Revisit usage of deprecated constructor as per TSK-583
300  //BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, "Known Bad", hashSetName);
301  BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, hashSetName);
302  badFile.addAttribute(att2);
303  BlackboardAttribute att3 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, MODULE_NAME, md5Hash);
304  badFile.addAttribute(att3);
305  BlackboardAttribute att4 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, comment);
306  badFile.addAttribute(att4);
307 
308  try {
309  // index the artifact for keyword search
310  blackboard.indexArtifact(badFile);
311  } catch (Blackboard.BlackboardException ex) {
312  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + badFile.getArtifactID(), ex); //NON-NLS
314  Bundle.HashDbIngestModule_indexError_message(), badFile.getDisplayName());
315  }
316 
317  if (showInboxMessage) {
318  StringBuilder detailsSb = new StringBuilder();
319  //details
320  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
321  //hit
322  detailsSb.append("<tr>"); //NON-NLS
323  detailsSb.append("<th>") //NON-NLS
324  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.fileName"))
325  .append("</th>"); //NON-NLS
326  detailsSb.append("<td>") //NON-NLS
327  .append(abstractFile.getName())
328  .append("</td>"); //NON-NLS
329  detailsSb.append("</tr>"); //NON-NLS
330 
331  detailsSb.append("<tr>"); //NON-NLS
332  detailsSb.append("<th>") //NON-NLS
333  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.md5Hash"))
334  .append("</th>"); //NON-NLS
335  detailsSb.append("<td>").append(md5Hash).append("</td>"); //NON-NLS
336  detailsSb.append("</tr>"); //NON-NLS
337 
338  detailsSb.append("<tr>"); //NON-NLS
339  detailsSb.append("<th>") //NON-NLS
340  .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.hashsetName"))
341  .append("</th>"); //NON-NLS
342  detailsSb.append("<td>").append(hashSetName).append("</td>"); //NON-NLS
343  detailsSb.append("</tr>"); //NON-NLS
344 
345  detailsSb.append("</table>"); //NON-NLS
346 
348  NbBundle.getMessage(this.getClass(),
349  "HashDbIngestModule.postToBB.knownBadMsg",
350  abstractFile.getName()),
351  detailsSb.toString(),
352  abstractFile.getName() + md5Hash,
353  badFile));
354  }
355  services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_HASHSET_HIT, Collections.singletonList(badFile)));
356  } catch (TskException ex) {
357  logger.log(Level.WARNING, "Error creating blackboard artifact", ex); //NON-NLS
358  }
359  }
360 
361  private static synchronized void postSummary(long jobId,
362  List<HashDb> knownBadHashSets, List<HashDb> knownHashSets) {
363  IngestJobTotals jobTotals = getTotalsForIngestJobs(jobId);
364  totalsForIngestJobs.remove(jobId);
365 
366  if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
367  StringBuilder detailsSb = new StringBuilder();
368  //details
369  detailsSb.append("<table border='0' cellpadding='4' width='280'>"); //NON-NLS
370 
371  detailsSb.append("<tr><td>") //NON-NLS
372  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.knownBadsFound"))
373  .append("</td>"); //NON-NLS
374  detailsSb.append("<td>").append(jobTotals.totalKnownBadCount.get()).append("</td></tr>"); //NON-NLS
375 
376  detailsSb.append("<tr><td>") //NON-NLS
377  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalCalcTime"))
378  .append("</td><td>").append(jobTotals.totalCalctime.get()).append("</td></tr>\n"); //NON-NLS
379  detailsSb.append("<tr><td>") //NON-NLS
380  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.totalLookupTime"))
381  .append("</td><td>").append(jobTotals.totalLookuptime.get()).append("</td></tr>\n"); //NON-NLS
382  detailsSb.append("</table>"); //NON-NLS
383 
384  detailsSb.append("<p>") //NON-NLS
385  .append(NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.complete.databasesUsed"))
386  .append("</p>\n<ul>"); //NON-NLS
387  for (HashDb db : knownBadHashSets) {
388  detailsSb.append("<li>").append(db.getHashSetName()).append("</li>\n"); //NON-NLS
389  }
390 
391  detailsSb.append("</ul>"); //NON-NLS
392 
395  HashLookupModuleFactory.getModuleName(),
396  NbBundle.getMessage(HashDbIngestModule.class,
397  "HashDbIngestModule.complete.hashLookupResults"),
398  detailsSb.toString()));
399  }
400  }
401 
402  @Override
403  public void shutDown() {
404  if (refCounter.decrementAndGet(jobId) == 0) {
405  postSummary(jobId, knownBadHashSets, knownHashSets);
406  }
407  }
408 }
static String calculateMd5(AbstractFile file)
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
TskData.TSK_DB_FILES_TYPE_ENUM getType()
void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context)
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId)
void updateEnabledHashSets(List< HashDb > allHashSets, List< HashDb > enabledHashSets)
void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage)
void addAttribute(BlackboardAttribute attr)
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
static synchronized void postSummary(long jobId, List< HashDb > knownBadHashSets, List< HashDb > knownHashSets)
BlackboardArtifact newArtifact(int artifactTypeID)
synchronized void indexArtifact(BlackboardArtifact artifact)
Definition: Blackboard.java:59
boolean setKnown(AbstractFile file, FileKnown fileKnown)
synchronized static Logger getLogger(String name)
Definition: Logger.java:161
static IngestMessage createWarningMessage(String source, String subject, String detailsHtml)
static synchronized IngestServices getInstance()

Copyright © 2012-2016 Basis Technology. Generated on: Mon Apr 24 2017
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.