Autopsy  4.17.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
DiscoveryAttributes.java
Go to the documentation of this file.
1 /*
2  * Autopsy
3  *
4  * Copyright 2020 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.discovery.search;
20 
21 import java.sql.ResultSet;
22 import java.sql.SQLException;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.Iterator;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.logging.Level;
32 import org.openide.util.NbBundle;
40 import org.sleuthkit.datamodel.BlackboardArtifact;
41 import org.sleuthkit.datamodel.BlackboardAttribute;
42 import org.sleuthkit.datamodel.CaseDbAccessManager;
43 import org.sleuthkit.datamodel.ContentTag;
44 import org.sleuthkit.datamodel.SleuthkitCase;
45 import org.sleuthkit.datamodel.TskCoreException;
46 import org.sleuthkit.datamodel.TskData;
47 import java.util.StringJoiner;
49 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_CATEGORIZATION;
50 
55 public class DiscoveryAttributes {
56 
57  private final static Logger logger = Logger.getLogger(DiscoveryAttributes.class.getName());
58 
62  public abstract static class AttributeType {
63 
72  public abstract DiscoveryKeyUtils.GroupKey getGroupKey(Result result);
73 
84  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb, CentralRepository centralRepoDb) throws DiscoveryException {
85  // Default is to do nothing
86  }
87  }
88 
92  public static class FileSizeAttribute extends AttributeType {
93 
94  @Override
96  return new DiscoveryKeyUtils.FileSizeGroupKey(result);
97  }
98  }
99 
103  public static class ParentPathAttribute extends AttributeType {
104 
105  @Override
108  }
109  }
110 
114  static class NoGroupingAttribute extends AttributeType {
115 
116  @Override
117  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
119  }
120  }
121 
125  static class DataSourceAttribute extends AttributeType {
126 
127  @Override
128  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
129  return new DiscoveryKeyUtils.DataSourceGroupKey(result);
130  }
131  }
132 
136  static class FileTypeAttribute extends AttributeType {
137 
138  @Override
139  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
140  return new DiscoveryKeyUtils.FileTypeGroupKey(file);
141  }
142  }
143 
148  static class DomainCategoryAttribute extends AttributeType {
149 
150  @Override
151  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
152  return new DiscoveryKeyUtils.DomainCategoryGroupKey(result);
153  }
154 
155  @Override
156  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
157  CentralRepository centralRepoDb) throws DiscoveryException {
158  try {
159  Map<String, String> domainsToCategories = getDomainsWithWebCategories(caseDb);
160  for (Result result : results) {
161  if (result instanceof ResultDomain) {
162  ResultDomain domain = (ResultDomain) result;
163  String webCategory = domainsToCategories.get(domain.getDomain());
164  domain.setWebCategory(webCategory);
165  }
166  }
167  } catch (TskCoreException | InterruptedException ex) {
168  throw new DiscoveryException("Error fetching TSK_WEB_CATEGORY artifacts from the database", ex);
169  }
170  }
171 
177  private Map<String, String> getDomainsWithWebCategories(SleuthkitCase caseDb) throws TskCoreException, InterruptedException {
178  Map<String, String> domainToCategory = new HashMap<>();
179 
180  for (BlackboardArtifact artifact : caseDb.getBlackboardArtifacts(TSK_WEB_CATEGORIZATION)) {
181  if (Thread.currentThread().isInterrupted()) {
182  throw new InterruptedException();
183  }
184 
185  BlackboardAttribute webCategory = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME));
186  BlackboardAttribute domain = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN));
187 
188  if (webCategory != null && domain != null) {
189  String webCatDisplayName = webCategory.getValueString();
190  String domainDisplayName = domain.getValueString().trim().toLowerCase();
191  domainToCategory.put(domainDisplayName, webCatDisplayName);
192  }
193  }
194 
195  return domainToCategory;
196  }
197  }
198 
202  static class KeywordListAttribute extends AttributeType {
203 
204  @Override
205  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
206  return new DiscoveryKeyUtils.KeywordListGroupKey((ResultFile) file);
207  }
208 
209  @Override
210  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
211  CentralRepository centralRepoDb) throws DiscoveryException {
212 
213  // Get pairs of (object ID, keyword list name) for all files in the list of files that have
214  // keyword list hits.
215  String selectQuery = createSetNameClause(results, BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID(),
216  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID());
217 
218  SetKeywordListNamesCallback callback = new SetKeywordListNamesCallback(results);
219  try {
220  caseDb.getCaseDbAccessManager().select(selectQuery, callback);
221  } catch (TskCoreException ex) {
222  throw new DiscoveryException("Error looking up keyword list attributes", ex); // NON-NLS
223  }
224  }
225 
231  private static class SetKeywordListNamesCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback {
232 
233  List<Result> resultFiles;
234 
240  SetKeywordListNamesCallback(List<Result> resultFiles) {
241  this.resultFiles = resultFiles;
242  }
243 
244  @Override
245  public void process(ResultSet rs) {
246  try {
247  // Create a temporary map of object ID to ResultFile
248  Map<Long, ResultFile> tempMap = new HashMap<>();
249  for (Result result : resultFiles) {
250  if (result.getType() == SearchData.Type.DOMAIN) {
251  break;
252  }
253  ResultFile file = (ResultFile) result;
254  tempMap.put(file.getFirstInstance().getId(), file);
255  }
256 
257  while (rs.next()) {
258  try {
259  Long objId = rs.getLong("object_id"); // NON-NLS
260  String keywordListName = rs.getString("set_name"); // NON-NLS
261 
262  tempMap.get(objId).addKeywordListName(keywordListName);
263 
264  } catch (SQLException ex) {
265  logger.log(Level.SEVERE, "Unable to get object_id or set_name from result set", ex); // NON-NLS
266  }
267  }
268  } catch (SQLException ex) {
269  logger.log(Level.SEVERE, "Failed to get keyword list names", ex); // NON-NLS
270  }
271  }
272  }
273  }
274 
284  private static Map<String, List<ResultDomain>> organizeByValue(List<ResultDomain> domainsBatch, CorrelationAttributeInstance.Type attributeType) {
285  final Map<String, List<ResultDomain>> resultDomainTable = new HashMap<>();
286  for (ResultDomain domainInstance : domainsBatch) {
287  try {
288  final String domainValue = domainInstance.getDomain();
289  final String normalizedDomain = CorrelationAttributeNormalizer.normalize(attributeType, domainValue);
290  final List<ResultDomain> bucket = resultDomainTable.getOrDefault(normalizedDomain, new ArrayList<>());
291  bucket.add(domainInstance);
292  resultDomainTable.put(normalizedDomain, bucket);
294  logger.log(Level.INFO, String.format("Domain [%s] failed normalization, skipping...", domainInstance.getDomain()));
295  }
296  }
297  return resultDomainTable;
298  }
299 
305  private static String createCSV(Set<String> values) {
306  StringJoiner joiner = new StringJoiner(", ");
307  for (String value : values) {
308  joiner.add("'" + value + "'");
309  }
310  return joiner.toString();
311  }
312 
316  static class PreviouslyNotableAttribute extends AttributeType {
317 
318  static final int DOMAIN_BATCH_SIZE = 500; // Number of domains to look up at one time
319 
320  @Override
321  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
322  return new DiscoveryKeyUtils.PreviouslyNotableGroupKey(result);
323  }
324 
325  @Override
326  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
327  CentralRepository centralRepoDb) throws DiscoveryException {
328 
329  if (centralRepoDb != null) {
330  processFilesWithCr(results, centralRepoDb);
331  }
332  }
333 
334  private void processFilesWithCr(List<Result> results, CentralRepository centralRepo) throws DiscoveryException {
335 
336  List<ResultDomain> domainsBatch = new ArrayList<>();
337  for (Result result : results) {
338  if (result.getType() == SearchData.Type.DOMAIN) {
339  domainsBatch.add((ResultDomain) result);
340  if (domainsBatch.size() == DOMAIN_BATCH_SIZE) {
341  queryPreviouslyNotable(domainsBatch, centralRepo);
342  domainsBatch.clear();
343  }
344  }
345  }
346 
347  queryPreviouslyNotable(domainsBatch, centralRepo);
348  }
349 
350  private void queryPreviouslyNotable(List<ResultDomain> domainsBatch, CentralRepository centralRepo) throws DiscoveryException {
351  if (domainsBatch.isEmpty()) {
352  return;
353  }
354 
355  try {
356  final CorrelationAttributeInstance.Type attributeType = centralRepo.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID);
357  final Map<String, List<ResultDomain>> resultDomainTable = organizeByValue(domainsBatch, attributeType);
358  final String values = createCSV(resultDomainTable.keySet());
359 
360  final String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType);
361  final String domainFrequencyQuery = " value AS domain_name "
362  + "FROM " + tableName + " "
363  + "WHERE value IN (" + values + ") "
364  + "AND known_status = " + TskData.FileKnown.BAD.getFileKnownValue();
365 
366  final DomainPreviouslyNotableCallback previouslyNotableCallback = new DomainPreviouslyNotableCallback(resultDomainTable);
367  centralRepo.processSelectClause(domainFrequencyQuery, previouslyNotableCallback);
368 
369  if (previouslyNotableCallback.getCause() != null) {
370  throw previouslyNotableCallback.getCause();
371  }
372  } catch (CentralRepoException | SQLException ex) {
373  throw new DiscoveryException("Fatal exception encountered querying the CR.", ex);
374  }
375  }
376 
377  private static class DomainPreviouslyNotableCallback implements InstanceTableCallback {
378 
379  private final Map<String, List<ResultDomain>> domainLookup;
380  private SQLException sqlCause;
381 
382  private DomainPreviouslyNotableCallback(Map<String, List<ResultDomain>> domainLookup) {
383  this.domainLookup = domainLookup;
384  }
385 
386  @Override
387  public void process(ResultSet resultSet) {
388  try {
389  while (resultSet.next()) {
390  String domain = resultSet.getString("domain_name");
391  List<ResultDomain> domainInstances = domainLookup.get(domain);
392  for (ResultDomain domainInstance : domainInstances) {
393  domainInstance.markAsPreviouslyNotableInCR();
394  }
395  }
396  } catch (SQLException ex) {
397  this.sqlCause = ex;
398  }
399  }
400 
404  SQLException getCause() {
405  return this.sqlCause;
406  }
407  }
408  }
409 
413  static class FrequencyAttribute extends AttributeType {
414 
415  static final int BATCH_SIZE = 50; // Number of hashes to look up at one time
416 
417  static final int DOMAIN_BATCH_SIZE = 500; // Number of domains to look up at one time
418 
419  @Override
420  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
421  return new DiscoveryKeyUtils.FrequencyGroupKey(file);
422  }
423 
424  @Override
425  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
426  CentralRepository centralRepoDb) throws DiscoveryException {
427  if (centralRepoDb == null) {
428  for (Result result : results) {
429  if (result.getFrequency() == SearchData.Frequency.UNKNOWN && result.getKnown() == TskData.FileKnown.KNOWN) {
430  result.setFrequency(SearchData.Frequency.KNOWN);
431  }
432  }
433  } else {
434  processResultFilesForCR(results, centralRepoDb);
435  }
436  }
437 
446  private void processResultFilesForCR(List<Result> results,
447  CentralRepository centralRepoDb) throws DiscoveryException {
448  List<ResultFile> currentFiles = new ArrayList<>();
449  Set<String> hashesToLookUp = new HashSet<>();
450  List<ResultDomain> domainsToQuery = new ArrayList<>();
451  for (Result result : results) {
452  // If frequency was already calculated, skip...
453  if (result.getFrequency() == SearchData.Frequency.UNKNOWN) {
454  if (result.getKnown() == TskData.FileKnown.KNOWN) {
455  result.setFrequency(SearchData.Frequency.KNOWN);
456  }
457 
458  if (result.getType() != SearchData.Type.DOMAIN) {
459  ResultFile file = (ResultFile) result;
460  if (file.getFirstInstance().getMd5Hash() != null
461  && !file.getFirstInstance().getMd5Hash().isEmpty()) {
462  hashesToLookUp.add(file.getFirstInstance().getMd5Hash());
463  currentFiles.add(file);
464  }
465 
466  if (hashesToLookUp.size() >= BATCH_SIZE) {
467  computeFrequency(hashesToLookUp, currentFiles, centralRepoDb);
468 
469  hashesToLookUp.clear();
470  currentFiles.clear();
471  }
472  } else {
473  domainsToQuery.add((ResultDomain) result);
474  if (domainsToQuery.size() == DOMAIN_BATCH_SIZE) {
475  queryDomainFrequency(domainsToQuery, centralRepoDb);
476 
477  domainsToQuery.clear();
478  }
479  }
480  }
481  }
482 
483  queryDomainFrequency(domainsToQuery, centralRepoDb);
484  computeFrequency(hashesToLookUp, currentFiles, centralRepoDb);
485  }
486  }
487 
496  private static void queryDomainFrequency(List<ResultDomain> domainsToQuery, CentralRepository centralRepository) throws DiscoveryException {
497  if (domainsToQuery.isEmpty()) {
498  return;
499  }
500  try {
501  final CorrelationAttributeInstance.Type attributeType = centralRepository.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID);
502  final Map<String, List<ResultDomain>> resultDomainTable = organizeByValue(domainsToQuery, attributeType);
503  final String values = createCSV(resultDomainTable.keySet());
504  final String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType);
505  final String domainFrequencyQuery = " value AS domain_name, COUNT(value) AS frequency FROM"
506  + "(SELECT DISTINCT case_id, value FROM "
507  + tableName
508  + " WHERE value IN ("
509  + values
510  + ")) AS foo GROUP BY value";
511 
512  final DomainFrequencyCallback frequencyCallback = new DomainFrequencyCallback(resultDomainTable);
513  centralRepository.processSelectClause(domainFrequencyQuery, frequencyCallback);
514 
515  if (frequencyCallback.getCause() != null) {
516  throw frequencyCallback.getCause();
517  }
518  } catch (CentralRepoException | SQLException ex) {
519  throw new DiscoveryException("Fatal exception encountered querying the CR.", ex);
520  }
521  }
522 
526  private static class DomainFrequencyCallback implements InstanceTableCallback {
527 
528  private final Map<String, List<ResultDomain>> domainLookup;
529  private SQLException sqlCause;
530 
536  private DomainFrequencyCallback(Map<String, List<ResultDomain>> domainLookup) {
537  this.domainLookup = domainLookup;
538  }
539 
540  @Override
541  public void process(ResultSet resultSet) {
542  try {
543  while (resultSet.next()) {
544  String domain = resultSet.getString("domain_name");
545  Long frequency = resultSet.getLong("frequency");
546 
547  List<ResultDomain> domainInstances = domainLookup.get(domain);
548  for (ResultDomain domainInstance : domainInstances) {
549  domainInstance.setFrequency(SearchData.Frequency.fromCount(frequency));
550  }
551  }
552  } catch (SQLException ex) {
553  this.sqlCause = ex;
554  }
555  }
556 
562  SQLException getCause() {
563  return this.sqlCause;
564  }
565  }
566 
571  private static class FrequencyCallback implements InstanceTableCallback {
572 
573  private final List<ResultFile> files;
574 
580  private FrequencyCallback(List<ResultFile> files) {
581  this.files = new ArrayList<>(files);
582  }
583 
584  @Override
585  public void process(ResultSet resultSet) {
586  try {
587 
588  while (resultSet.next()) {
589  String hash = resultSet.getString(1);
590  int count = resultSet.getInt(2);
591  for (Iterator<ResultFile> iterator = files.iterator(); iterator.hasNext();) {
592  ResultFile file = iterator.next();
593  if (file.getFirstInstance().getMd5Hash().equalsIgnoreCase(hash)) {
595  iterator.remove();
596  }
597  }
598  }
599 
600  // The files left had no matching entries in the CR, so mark them as unique
601  for (ResultFile file : files) {
602  file.setFrequency(SearchData.Frequency.UNIQUE);
603  }
604  } catch (SQLException ex) {
605  logger.log(Level.WARNING, "Error getting frequency counts from Central Repository", ex); // NON-NLS
606  }
607  }
608  }
609 
613  static class HashHitsAttribute extends AttributeType {
614 
615  @Override
616  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
617  if (result.getType() == SearchData.Type.DOMAIN) {
618  return null;
619  }
620  return new DiscoveryKeyUtils.HashHitsGroupKey((ResultFile) result);
621  }
622 
623  @Override
624  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
625  CentralRepository centralRepoDb) throws DiscoveryException {
626 
627  // Get pairs of (object ID, hash set name) for all files in the list of files that have
628  // hash set hits.
629  String selectQuery = createSetNameClause(results, BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID(),
630  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID());
631 
632  HashSetNamesCallback callback = new HashSetNamesCallback(results);
633  try {
634  caseDb.getCaseDbAccessManager().select(selectQuery, callback);
635  } catch (TskCoreException ex) {
636  throw new DiscoveryException("Error looking up hash set attributes", ex); // NON-NLS
637  }
638  }
639 
644  private static class HashSetNamesCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback {
645 
646  List<Result> results;
647 
653  HashSetNamesCallback(List<Result> results) {
654  this.results = results;
655  }
656 
657  @Override
658  public void process(ResultSet rs) {
659  try {
660  // Create a temporary map of object ID to ResultFile
661  Map<Long, ResultFile> tempMap = new HashMap<>();
662  for (Result result : results) {
663  if (result.getType() == SearchData.Type.DOMAIN) {
664  return;
665  }
666  ResultFile file = (ResultFile) result;
667  tempMap.put(file.getFirstInstance().getId(), file);
668  }
669 
670  while (rs.next()) {
671  try {
672  Long objId = rs.getLong("object_id"); // NON-NLS
673  String hashSetName = rs.getString("set_name"); // NON-NLS
674 
675  tempMap.get(objId).addHashSetName(hashSetName);
676 
677  } catch (SQLException ex) {
678  logger.log(Level.SEVERE, "Unable to get object_id or set_name from result set", ex); // NON-NLS
679  }
680  }
681  } catch (SQLException ex) {
682  logger.log(Level.SEVERE, "Failed to get hash set names", ex); // NON-NLS
683  }
684  }
685  }
686  }
687 
691  static class InterestingItemAttribute extends AttributeType {
692 
693  @Override
694  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
696  }
697 
698  @Override
699  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
700  CentralRepository centralRepoDb) throws DiscoveryException {
701 
702  // Get pairs of (object ID, interesting item set name) for all files in the list of files that have
703  // interesting file set hits.
704  String selectQuery = createSetNameClause(results, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT.getTypeID(),
705  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID());
706 
707  InterestingFileSetNamesCallback callback = new InterestingFileSetNamesCallback(results);
708  try {
709  caseDb.getCaseDbAccessManager().select(selectQuery, callback);
710  } catch (TskCoreException ex) {
711  throw new DiscoveryException("Error looking up interesting file set attributes", ex); // NON-NLS
712  }
713  }
714 
720  private static class InterestingFileSetNamesCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback {
721 
722  List<Result> results;
723 
730  InterestingFileSetNamesCallback(List<Result> results) {
731  this.results = results;
732  }
733 
734  @Override
735  public void process(ResultSet rs) {
736  try {
737  // Create a temporary map of object ID to ResultFile
738  Map<Long, ResultFile> tempMap = new HashMap<>();
739  for (Result result : results) {
740  if (result.getType() == SearchData.Type.DOMAIN) {
741  return;
742  }
743  ResultFile file = (ResultFile) result;
744  tempMap.put(file.getFirstInstance().getId(), file);
745  }
746 
747  while (rs.next()) {
748  try {
749  Long objId = rs.getLong("object_id"); // NON-NLS
750  String setName = rs.getString("set_name"); // NON-NLS
751 
752  tempMap.get(objId).addInterestingSetName(setName);
753 
754  } catch (SQLException ex) {
755  logger.log(Level.SEVERE, "Unable to get object_id or set_name from result set", ex); // NON-NLS
756  }
757  }
758  } catch (SQLException ex) {
759  logger.log(Level.SEVERE, "Failed to get interesting file set names", ex); // NON-NLS
760  }
761  }
762  }
763  }
764 
768  static class LastActivityDateAttribute extends AttributeType {
769 
770  @Override
771  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
772  return new DiscoveryKeyUtils.LastActivityDateGroupKey(result);
773  }
774 
775  }
776 
780  static class FirstActivityDateAttribute extends AttributeType {
781 
782  @Override
783  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
784  return new DiscoveryKeyUtils.FirstActivityDateGroupKey(result);
785  }
786 
787  }
788 
793  static class PageViewsAttribute extends AttributeType {
794 
795  @Override
796  public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
797  return new DiscoveryKeyUtils.PageViewsGroupKey(result);
798  }
799  }
800 
804  static class ObjectDetectedAttribute extends AttributeType {
805 
806  @Override
807  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
808  return new DiscoveryKeyUtils.ObjectDetectedGroupKey((ResultFile) file);
809  }
810 
811  @Override
812  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
813  CentralRepository centralRepoDb) throws DiscoveryException {
814 
815  // Get pairs of (object ID, object type name) for all files in the list of files that have
816  // objects detected
817  String selectQuery = createSetNameClause(results, BlackboardArtifact.ARTIFACT_TYPE.TSK_OBJECT_DETECTED.getTypeID(),
818  BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION.getTypeID());
819 
820  ObjectDetectedNamesCallback callback = new ObjectDetectedNamesCallback(results);
821  try {
822  caseDb.getCaseDbAccessManager().select(selectQuery, callback);
823  } catch (TskCoreException ex) {
824  throw new DiscoveryException("Error looking up object detected attributes", ex); // NON-NLS
825  }
826  }
827 
833  private static class ObjectDetectedNamesCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback {
834 
835  List<Result> results;
836 
842  ObjectDetectedNamesCallback(List<Result> results) {
843  this.results = results;
844  }
845 
846  @Override
847  public void process(ResultSet rs) {
848  try {
849  // Create a temporary map of object ID to ResultFile
850  Map<Long, ResultFile> tempMap = new HashMap<>();
851  for (Result result : results) {
852  if (result.getType() == SearchData.Type.DOMAIN) {
853  return;
854  }
855  ResultFile file = (ResultFile) result;
856  tempMap.put(file.getFirstInstance().getId(), file);
857  }
858 
859  while (rs.next()) {
860  try {
861  Long objId = rs.getLong("object_id"); // NON-NLS
862  String setName = rs.getString("set_name"); // NON-NLS
863 
864  tempMap.get(objId).addObjectDetectedName(setName);
865 
866  } catch (SQLException ex) {
867  logger.log(Level.SEVERE, "Unable to get object_id or set_name from result set", ex); // NON-NLS
868  }
869  }
870  } catch (SQLException ex) {
871  logger.log(Level.SEVERE, "Failed to get object detected names", ex); // NON-NLS
872  }
873  }
874  }
875  }
876 
880  static class FileTagAttribute extends AttributeType {
881 
882  @Override
883  public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) {
884  return new DiscoveryKeyUtils.FileTagGroupKey((ResultFile) file);
885  }
886 
887  @Override
888  public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
889  CentralRepository centralRepoDb) throws DiscoveryException {
890 
891  try {
892  for (Result result : results) {
893  if (result.getType() == SearchData.Type.DOMAIN) {
894  return;
895  }
896  ResultFile file = (ResultFile) result;
897  List<ContentTag> contentTags = caseDb.getContentTagsByContent(file.getFirstInstance());
898 
899  for (ContentTag tag : contentTags) {
900  result.addTagName(tag.getName().getDisplayName());
901  }
902  }
903  } catch (TskCoreException ex) {
904  throw new DiscoveryException("Error looking up file tag attributes", ex); // NON-NLS
905  }
906  }
907  }
908 
912  @NbBundle.Messages({
913  "DiscoveryAttributes.GroupingAttributeType.fileType.displayName=File Type",
914  "DiscoveryAttributes.GroupingAttributeType.frequency.displayName=Past Occurrences",
915  "DiscoveryAttributes.GroupingAttributeType.keywordList.displayName=Keyword",
916  "DiscoveryAttributes.GroupingAttributeType.size.displayName=File Size",
917  "DiscoveryAttributes.GroupingAttributeType.datasource.displayName=Data Source",
918  "DiscoveryAttributes.GroupingAttributeType.parent.displayName=Parent Folder",
919  "DiscoveryAttributes.GroupingAttributeType.hash.displayName=Hash Set",
920  "DiscoveryAttributes.GroupingAttributeType.interestingItem.displayName=Interesting Item",
921  "DiscoveryAttributes.GroupingAttributeType.tag.displayName=Tag",
922  "DiscoveryAttributes.GroupingAttributeType.object.displayName=Object Detected",
923  "DiscoveryAttributes.GroupingAttributeType.lastDate.displayName=Last Activity Date",
924  "DiscoveryAttributes.GroupingAttributeType.firstDate.displayName=First Activity Date",
925  "DiscoveryAttributes.GroupingAttributeType.pageViews.displayName=Page Views",
926  "DiscoveryAttributes.GroupingAttributeType.none.displayName=None",
927  "DiscoveryAttributes.GroupingAttributeType.previouslyNotable.displayName=Previous Notability",
928  "DiscoveryAttributes.GroupingAttributeType.webCategory.displayName=Domain Category"})
929  public enum GroupingAttributeType {
930  FILE_SIZE(new FileSizeAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_size_displayName()),
931  FREQUENCY(new FrequencyAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_frequency_displayName()),
932  KEYWORD_LIST_NAME(new KeywordListAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_keywordList_displayName()),
933  DATA_SOURCE(new DataSourceAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_datasource_displayName()),
934  PARENT_PATH(new ParentPathAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_parent_displayName()),
935  HASH_LIST_NAME(new HashHitsAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_hash_displayName()),
936  INTERESTING_ITEM_SET(new InterestingItemAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_interestingItem_displayName()),
937  FILE_TAG(new FileTagAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_tag_displayName()),
938  OBJECT_DETECTED(new ObjectDetectedAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_object_displayName()),
939  LAST_ACTIVITY_DATE(new LastActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_lastDate_displayName()),
940  FIRST_ACTIVITY_DATE(new FirstActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_firstDate_displayName()),
941  PAGE_VIEWS(new PageViewsAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_pageViews_displayName()),
942  NO_GROUPING(new NoGroupingAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_none_displayName()),
943  PREVIOUSLY_NOTABLE(new PreviouslyNotableAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_previouslyNotable_displayName()),
944  DOMAIN_CATEGORY(new DomainCategoryAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_webCategory_displayName());
945 
947  private final String displayName;
948 
957  GroupingAttributeType(AttributeType attributeType, String displayName) {
958  this.attributeType = attributeType;
959  this.displayName = displayName;
960  }
961 
962  @Override
963  public String toString() {
964  return displayName;
965  }
966 
973  return attributeType;
974  }
975 
981  public static List<GroupingAttributeType> getOptionsForGroupingForFiles() {
982  return Arrays.asList(FILE_SIZE, FREQUENCY, PARENT_PATH, OBJECT_DETECTED, HASH_LIST_NAME, INTERESTING_ITEM_SET);
983  }
984 
990  public static List<GroupingAttributeType> getOptionsForGroupingForDomains() {
992  return Arrays.asList(PAGE_VIEWS, FREQUENCY, LAST_ACTIVITY_DATE, FIRST_ACTIVITY_DATE, PREVIOUSLY_NOTABLE, DOMAIN_CATEGORY);
993  } else {
994  return Arrays.asList(PAGE_VIEWS, LAST_ACTIVITY_DATE, FIRST_ACTIVITY_DATE, DOMAIN_CATEGORY);
995  }
996  }
997  }
998 
1007  private static void computeFrequency(Set<String> hashesToLookUp, List<ResultFile> currentFiles, CentralRepository centralRepoDb) {
1008 
1009  if (hashesToLookUp.isEmpty()) {
1010  return;
1011  }
1012 
1013  String hashes = String.join("','", hashesToLookUp);
1014  hashes = "'" + hashes + "'";
1015  try {
1017  String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType);
1018 
1019  String selectClause = " value, COUNT(value) FROM "
1020  + "(SELECT DISTINCT case_id, value FROM " + tableName
1021  + " WHERE value IN ("
1022  + hashes
1023  + ")) AS foo GROUP BY value";
1024 
1025  FrequencyCallback callback = new FrequencyCallback(currentFiles);
1026  centralRepoDb.processSelectClause(selectClause, callback);
1027 
1028  } catch (CentralRepoException ex) {
1029  logger.log(Level.WARNING, "Error getting frequency counts from Central Repository", ex); // NON-NLS
1030  }
1031 
1032  }
1033 
1047  private static String createSetNameClause(List<Result> results,
1048  int artifactTypeID, int setNameAttrID) throws DiscoveryException {
1049 
1050  // Concatenate the object IDs in the list of files
1051  String objIdList = ""; // NON-NLS
1052  for (Result result : results) {
1053  if (result.getType() == SearchData.Type.DOMAIN) {
1054  break;
1055  }
1056  ResultFile file = (ResultFile) result;
1057  if (!objIdList.isEmpty()) {
1058  objIdList += ","; // NON-NLS
1059  }
1060  objIdList += "\'" + file.getFirstInstance().getId() + "\'"; // NON-NLS
1061  }
1062 
1063  // Get pairs of (object ID, set name) for all files in the list of files that have
1064  // the given artifact type.
1065  return "blackboard_artifacts.obj_id AS object_id, blackboard_attributes.value_text AS set_name "
1066  + "FROM blackboard_artifacts "
1067  + "INNER JOIN blackboard_attributes ON blackboard_artifacts.artifact_id=blackboard_attributes.artifact_id "
1068  + "WHERE blackboard_attributes.artifact_type_id=\'" + artifactTypeID + "\' "
1069  + "AND blackboard_attributes.attribute_type_id=\'" + setNameAttrID + "\' "
1070  + "AND blackboard_artifacts.obj_id IN (" + objIdList
1071  + ") "; // NON-NLS
1072  }
1073 
1078  // Class should not be instantiated
1079  }
1080 }
static String createSetNameClause(List< Result > results, int artifactTypeID, int setNameAttrID)
abstract TskData.FileKnown getKnown()
static void computeFrequency(Set< String > hashesToLookUp, List< ResultFile > currentFiles, CentralRepository centralRepoDb)
final void setFrequency(SearchData.Frequency frequency)
Definition: Result.java:81
static String correlationTypeToInstanceTableName(CorrelationAttributeInstance.Type type)
static String normalize(CorrelationAttributeInstance.Type attributeType, String data)
void addAttributeToResults(List< Result > results, SleuthkitCase caseDb, CentralRepository centralRepoDb)
abstract DiscoveryKeyUtils.GroupKey getGroupKey(Result result)
void addKeywordListName(String keywordListName)
static Map< String, List< ResultDomain > > organizeByValue(List< ResultDomain > domainsBatch, CorrelationAttributeInstance.Type attributeType)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
CorrelationAttributeInstance.Type getCorrelationTypeById(int typeId)
void processSelectClause(String selectClause, InstanceTableCallback instanceTableCallback)
static void queryDomainFrequency(List< ResultDomain > domainsToQuery, CentralRepository centralRepository)

Copyright © 2012-2021 Basis Technology. Generated on: Tue Jan 19 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.