Autopsy  4.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
KeywordSearchResultFactory.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2016 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this content except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.awt.EventQueue;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.LinkedHashMap;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.concurrent.CancellationException;
29 import java.util.concurrent.ExecutionException;
30 import java.util.logging.Level;
31 import javax.swing.SwingWorker;
32 import org.netbeans.api.progress.ProgressHandle;
33 import org.openide.nodes.ChildFactory;
34 import org.openide.nodes.Children;
35 import org.openide.nodes.Node;
36 import org.openide.util.NbBundle;
37 import org.openide.util.lookup.Lookups;
46 import org.sleuthkit.datamodel.AbstractFile;
47 import org.sleuthkit.datamodel.BlackboardArtifact;
48 import org.sleuthkit.datamodel.BlackboardAttribute;
49 import org.sleuthkit.datamodel.Content;
50 
58 class KeywordSearchResultFactory extends ChildFactory<KeyValueQueryContent> {
59 
60  //common properties (superset of all Node properties) to be displayed as columns
61  //these are merged with FsContentPropertyType defined properties
62  public static enum CommonPropertyTypes {
63 
64  KEYWORD {
65  @Override
66  public String toString() {
67  return BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getDisplayName();
68  }
69  },
70  REGEX {
71  @Override
72  public String toString() {
73  return BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getDisplayName();
74  }
75  },
76  CONTEXT {
77  @Override
78  public String toString() {
79  return BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getDisplayName();
80  }
81  },
82  }
83  private final Collection<QueryRequest> queryRequests;
84  private static final Logger logger = Logger.getLogger(KeywordSearchResultFactory.class.getName());
85 
86  KeywordSearchResultFactory(Collection<QueryRequest> queryRequests, DataResultTopComponent viewer) {
87  this.queryRequests = queryRequests;
88  }
89 
97  public static void initCommonProperties(Map<String, Object> toSet) {
98  CommonPropertyTypes[] commonTypes = CommonPropertyTypes.values();
99  final int COMMON_PROPS_LEN = commonTypes.length;
100  for (int i = 0; i < COMMON_PROPS_LEN; ++i) {
101  toSet.put(commonTypes[i].toString(), "");
102  }
103 
104  AbstractAbstractFileNode.AbstractFilePropertyType[] fsTypes = AbstractAbstractFileNode.AbstractFilePropertyType.values();
105  final int FS_PROPS_LEN = fsTypes.length;
106  for (int i = 0; i < FS_PROPS_LEN; ++i) {
107  toSet.put(fsTypes[i].toString(), "");
108  }
109  }
110 
111  public static void setCommonProperty(Map<String, Object> toSet, CommonPropertyTypes type, String value) {
112  final String typeStr = type.toString();
113  toSet.put(typeStr, value);
114  }
115 
116  public static void setCommonProperty(Map<String, Object> toSet, CommonPropertyTypes type, Boolean value) {
117  final String typeStr = type.toString();
118  toSet.put(typeStr, value);
119  }
120 
121  @Override
122  protected boolean createKeys(List<KeyValueQueryContent> toPopulate) {
123 
124  for (QueryRequest queryRequest : queryRequests) {
125  Map<String, Object> map = queryRequest.getProperties();
126  initCommonProperties(map);
127  final String query = queryRequest.getQueryString();
128  setCommonProperty(map, CommonPropertyTypes.KEYWORD, query);
129  setCommonProperty(map, CommonPropertyTypes.REGEX, !queryRequest.getQuery().isLiteral());
130  createFlatKeys(queryRequest, toPopulate);
131  }
132 
133  return true;
134  }
135 
143  @NbBundle.Messages({"KeywordSearchResultFactory.query.exception.msg=Could not perform the query "})
144  private boolean createFlatKeys(QueryRequest queryRequest, List<KeyValueQueryContent> toPopulate) {
148  final KeywordSearchQuery keywordSearchQuery = queryRequest.getQuery();
149  if (!keywordSearchQuery.validate()) {
150  //TODO mark the particular query node RED
151  return false;
152  }
153 
157  QueryResults queryResults;
158  try {
159  queryResults = keywordSearchQuery.performQuery();
160  } catch (KeywordSearchModuleException | NoOpenCoreException ex) {
161  logger.log(Level.SEVERE, "Could not perform the query " + keywordSearchQuery.getQueryString(), ex); //NON-NLS
162  MessageNotifyUtil.Notify.error(Bundle.KeywordSearchResultFactory_query_exception_msg() + keywordSearchQuery.getQueryString(), ex.getCause().getMessage());
163  return false;
164  }
165 
166  int id = 0;
167  List<KeyValueQueryContent> tempList = new ArrayList<>();
168  for (KeywordHit hit : getOneHitPerObject(queryResults)) {
172  Map<String, Object> properties = new LinkedHashMap<>();
173  Content content = hit.getContent();
174  if (content instanceof AbstractFile) {
175  AbstractFsContentNode.fillPropertyMap(properties, (AbstractFile) content);
176  } else {
177  properties.put(AbstractAbstractFileNode.AbstractFilePropertyType.LOCATION.toString(), content.getName());
178  }
179 
183  if (hit.hasSnippet()) {
184  setCommonProperty(properties, CommonPropertyTypes.CONTEXT, hit.getSnippet());
185  }
186 
187  //@@@ USE ConentHit in UniqueFileMap instead of the below search
188  //get unique match result files
189  // BC: @@@ THis is really ineffecient. We should keep track of this when
190  // we flattened the list of files to the unique files.
191  final String highlightQueryEscaped = getHighlightQuery(keywordSearchQuery, keywordSearchQuery.isLiteral(), queryResults, content);
192 
193  String name = content.getName();
194  if (hit.isArtifactHit()) {
195  name = hit.getArtifact().getDisplayName() + " Artifact"; // NON-NLS
196  }
197  ++id;
198  tempList.add(new KeyValueQueryContent(name, properties, id, hit.getSolrObjectId(), content, highlightQueryEscaped, keywordSearchQuery, queryResults));
199  }
200 
201  // Add all the nodes to toPopulate at once. Minimizes node creation
202  // EDT threads, which can slow and/or hang the UI on large queries.
203  toPopulate.addAll(tempList);
204 
205  //write to bb
206  //cannot reuse snippet in BlackboardResultWriter
207  //because for regex searches in UI we compress results by showing a content per regex once (even if multiple term hits)
208  //whereas in bb we write every hit per content separately
209  new BlackboardResultWriter(queryResults, queryRequest.getQuery().getKeywordList().getName()).execute();
210 
211  return true;
212  }
213 
223  Collection<KeywordHit> getOneHitPerObject(QueryResults queryResults) {
224  HashMap<Long, KeywordHit> hits = new HashMap<>();
225  for (Keyword keyWord : queryResults.getKeywords()) {
226  for (KeywordHit hit : queryResults.getResults(keyWord)) {
227  // add hit with lowest SolrObjectID-Chunk-ID combination.
228  if (!hits.containsKey(hit.getSolrObjectId())) {
229  hits.put(hit.getSolrObjectId(), hit);
230  } else {
231  if (hit.getChunkId() < hits.get(hit.getSolrObjectId()).getChunkId()) {
232  hits.put(hit.getSolrObjectId(), hit);
233  }
234  }
235  }
236  }
237  return hits.values();
238  }
239 
250  private String getHighlightQuery(KeywordSearchQuery query, boolean literal_query, QueryResults queryResults, Content content) {
251  if (literal_query) {
252  //literal, treat as non-regex, non-term component query
253  return constructEscapedSolrQuery(query.getQueryString(), literal_query);
254  } else {
255  //construct a Solr query using aggregated terms to get highlighting
256  //the query is executed later on demand
257  if (queryResults.getKeywords().size() == 1) {
258  //simple case, no need to process subqueries and do special escaping
259  Keyword keyword = queryResults.getKeywords().iterator().next();
260  return constructEscapedSolrQuery(keyword.getSearchTerm(), literal_query);
261  } else {
262  //find terms for this content hit
263  List<Keyword> hitTerms = new ArrayList<>();
264  for (Keyword keyword : queryResults.getKeywords()) {
265  for (KeywordHit hit : queryResults.getResults(keyword)) {
266  if (hit.getContent().equals(content)) {
267  hitTerms.add(keyword);
268  break; //go to next term
269  }
270  }
271  }
272 
273  StringBuilder highlightQuery = new StringBuilder();
274  final int lastTerm = hitTerms.size() - 1;
275  int curTerm = 0;
276  for (Keyword term : hitTerms) {
277  //escape subqueries, MAKE SURE they are not escaped again later
278  highlightQuery.append(constructEscapedSolrQuery(term.getSearchTerm(), literal_query));
279  if (lastTerm != curTerm) {
280  highlightQuery.append(" "); //acts as OR ||
281  }
282 
283  ++curTerm;
284  }
285  return highlightQuery.toString();
286  }
287  }
288  }
289 
297  private String constructEscapedSolrQuery(String query, boolean literal_query) {
298  StringBuilder highlightQuery = new StringBuilder();
299  String highLightField;
300  if (literal_query) {
301  highLightField = LuceneQuery.HIGHLIGHT_FIELD_LITERAL;
302  } else {
303  highLightField = LuceneQuery.HIGHLIGHT_FIELD_REGEX;
304  }
305  highlightQuery.append(highLightField).append(":").append("\"").append(KeywordSearchUtil.escapeLuceneQuery(query)).append("\"");
306  return highlightQuery.toString();
307  }
308 
309  @Override
310  protected Node createNodeForKey(KeyValueQueryContent key) {
311  final Content content = key.getContent();
312  final String queryStr = key.getQueryStr();
313  QueryResults hits = key.getHits();
314 
315  Node kvNode = new KeyValueNode(key, Children.LEAF, Lookups.singleton(content));
316 
317  //wrap in KeywordSearchFilterNode for the markup content, might need to override FilterNode for more customization
318  // store the data in HighlightedMatchesSource so that it can be looked up (in content viewer)
319  HighlightedText highlights = new HighlightedText(key.solrObjectId, queryStr, !key.getQuery().isLiteral(), false, hits);
320  return new KeywordSearchFilterNode(highlights, kvNode);
321  }
322 
327  class KeyValueQueryContent extends KeyValue {
328 
329  private long solrObjectId;
330  private final Content content;
331  private final String queryStr;
332  private final QueryResults hits;
333  private final KeywordSearchQuery query;
334 
350  public KeyValueQueryContent(String name, Map<String, Object> map, int id, long solrObjectId, Content content, String queryStr, KeywordSearchQuery query, QueryResults hits) {
351  super(name, map, id);
352  this.solrObjectId = solrObjectId;
353  this.content = content;
354  this.queryStr = queryStr;
355  this.hits = hits;
356  this.query = query;
357  }
358 
359  Content getContent() {
360  return content;
361  }
362 
363  String getQueryStr() {
364  return queryStr;
365  }
366 
367  QueryResults getHits() {
368  return hits;
369  }
370 
371  KeywordSearchQuery getQuery() {
372  return query;
373  }
374  }
375 
380  static class BlackboardResultWriter extends SwingWorker<Object, Void> {
381 
382  private static final List<BlackboardResultWriter> writers = new ArrayList<>();
383  private ProgressHandle progress;
384  private final KeywordSearchQuery query;
385  private final QueryResults hits;
386  private Collection<BlackboardArtifact> newArtifacts = new ArrayList<>();
387  private static final int QUERY_DISPLAY_LEN = 40;
388 
389  BlackboardResultWriter(QueryResults hits, String listName) {
390  this.hits = hits;
391  this.query = hits.getQuery();
392  }
393 
394  protected void finalizeWorker() {
395  deregisterWriter(this);
396  EventQueue.invokeLater(progress::finish);
397  }
398 
399  @Override
400  protected Object doInBackground() throws Exception {
401  registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock
402  final String queryStr = query.getQueryString();
403  final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr;
404  try {
405  progress = ProgressHandle.createHandle(NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), () -> BlackboardResultWriter.this.cancel(true));
406  newArtifacts = hits.writeAllHitsToBlackBoard(progress, null, this, false);
407  } finally {
408  finalizeWorker();
409  }
410  return null;
411  }
412 
413  @Override
414  protected void done() {
415  try {
416  get();
417  } catch (InterruptedException | CancellationException ex) {
418  logger.log(Level.WARNING, "User cancelled writing of ad hoc search query results for '{0}' to the blackboard", query.getQueryString()); //NON-NLS
419  } catch (ExecutionException ex) {
420  logger.log(Level.SEVERE, "Error writing of ad hoc search query results for " + query.getQueryString() + " to the blackboard", ex); //NON-NLS
421  }
422  }
423 
424  private static synchronized void registerWriter(BlackboardResultWriter writer) {
425  writers.add(writer);
426  }
427 
428  private static synchronized void deregisterWriter(BlackboardResultWriter writer) {
429  writers.remove(writer);
430  }
431 
432  static synchronized void stopAllWriters() {
433  for (BlackboardResultWriter w : writers) {
434  w.cancel(true);
435  writers.remove(w);
436  }
437  }
438  }
439 }
synchronized static Logger getLogger(String name)
Definition: Logger.java:161

Copyright © 2012-2016 Basis Technology. Generated on: Mon Jan 2 2017
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.