Autopsy  3.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
KeywordSearchResultFactory.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2015 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this content except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.awt.EventQueue;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.LinkedHashMap;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.concurrent.ExecutionException;
31 import java.util.concurrent.locks.ReentrantReadWriteLock;
32 import java.util.logging.Level;
33 import org.openide.util.NbBundle;
35 import javax.swing.SwingWorker;
36 import org.netbeans.api.progress.ProgressHandle;
37 import org.netbeans.api.progress.ProgressHandleFactory;
38 import org.openide.nodes.ChildFactory;
39 import org.openide.nodes.Children;
40 import org.openide.nodes.Node;
41 import org.openide.util.Cancellable;
42 import org.openide.util.lookup.Lookups;
53 
61 class KeywordSearchResultFactory extends ChildFactory<KeyValueQueryContent> {
62 
63  //common properties (superset of all Node properties) to be displayed as columns
64  //these are merged with FsContentPropertyType defined properties
65  public static enum CommonPropertyTypes {
66 
67  KEYWORD {
68  @Override
69  public String toString() {
70  return BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getDisplayName();
71  }
72  },
73  REGEX {
74  @Override
75  public String toString() {
77  }
78  },
79  CONTEXT {
80  @Override
81  public String toString() {
83  }
84  },
85  }
86  private Collection<QueryRequest> queryRequests;
87  private final DataResultTopComponent viewer; //viewer driving this child node factory
88  private static final Logger logger = Logger.getLogger(KeywordSearchResultFactory.class.getName());
89 
90  KeywordSearchResultFactory(Collection<QueryRequest> queryRequests, DataResultTopComponent viewer) {
91  this.queryRequests = queryRequests;
92  this.viewer = viewer;
93  }
94 
102  public static void initCommonProperties(Map<String, Object> toSet) {
103  CommonPropertyTypes[] commonTypes = CommonPropertyTypes.values();
104  final int COMMON_PROPS_LEN = commonTypes.length;
105  for (int i = 0; i < COMMON_PROPS_LEN; ++i) {
106  toSet.put(commonTypes[i].toString(), "");
107  }
108 
109  AbstractAbstractFileNode.AbstractFilePropertyType[] fsTypes = AbstractAbstractFileNode.AbstractFilePropertyType.values();
110  final int FS_PROPS_LEN = fsTypes.length;
111  for (int i = 0; i < FS_PROPS_LEN; ++i) {
112  toSet.put(fsTypes[i].toString(), "");
113  }
114  }
115 
116  public static void setCommonProperty(Map<String, Object> toSet, CommonPropertyTypes type, String value) {
117  final String typeStr = type.toString();
118  toSet.put(typeStr, value);
119  }
120 
121  public static void setCommonProperty(Map<String, Object> toSet, CommonPropertyTypes type, Boolean value) {
122  final String typeStr = type.toString();
123  toSet.put(typeStr, value);
124  }
125 
126  @Override
127  protected boolean createKeys(List<KeyValueQueryContent> toPopulate) {
128 
129  for (QueryRequest queryRequest : queryRequests) {
130  Map<String, Object> map = queryRequest.getProperties();
131  initCommonProperties(map);
132  final String query = queryRequest.getQueryString();
133  setCommonProperty(map, CommonPropertyTypes.KEYWORD, query);
134  setCommonProperty(map, CommonPropertyTypes.REGEX, !queryRequest.getQuery().isLiteral());
135  createFlatKeys(queryRequest, toPopulate);
136  }
137 
138  return true;
139  }
140 
147  private boolean createFlatKeys(QueryRequest queryRequest, List<KeyValueQueryContent> toPopulate) {
151  final KeywordSearchQuery keywordSearchQuery = queryRequest.getQuery();
152  if (!keywordSearchQuery.validate()) {
153  //TODO mark the particular query node RED
154  return false;
155  }
156 
160  QueryResults queryResults;
161  try {
162  queryResults = keywordSearchQuery.performQuery();
163  } catch (NoOpenCoreException ex) {
164  logger.log(Level.SEVERE, "Could not perform the query " + keywordSearchQuery.getQueryString(), ex); //NON-NLS
165  return false;
166  }
167 
168  int id = 0;
169  List<KeyValueQueryContent> tempList = new ArrayList<>();
170  for (KeywordHit hit : getOneHitPerObject(queryResults)) {
174  Map<String, Object> properties = new LinkedHashMap<>();
175  Content content = hit.getContent();
176  if (content instanceof AbstractFile) {
177  AbstractFsContentNode.fillPropertyMap(properties, (AbstractFile)content);
178  }
179  else {
180  properties.put(AbstractAbstractFileNode.AbstractFilePropertyType.LOCATION.toString(), content.getName());
181  }
182 
186  if (hit.hasSnippet()) {
187  setCommonProperty(properties, CommonPropertyTypes.CONTEXT, hit.getSnippet());
188  }
189 
190  //@@@ USE ConentHit in UniqueFileMap instead of the below search
191  //get unique match result files
192  // BC: @@@ THis is really ineffecient. We should keep track of this when
193  // we flattened the list of files to the unique files.
194  final String highlightQueryEscaped = getHighlightQuery(keywordSearchQuery, keywordSearchQuery.isLiteral(), queryResults, content);
195 
196  String name = content.getName();
197  if (hit.isArtifactHit())
198  name = hit.getArtifact().getDisplayName() + " Artifact"; // NON-NLS
199 
200  tempList.add(new KeyValueQueryContent(name, properties, ++id, hit.getSolrObjectId(), content, highlightQueryEscaped, keywordSearchQuery, queryResults));
201  }
202 
203  // Add all the nodes to toPopulate at once. Minimizes node creation
204  // EDT threads, which can slow and/or hang the UI on large queries.
205  toPopulate.addAll(tempList);
206 
207  //write to bb
208  //cannot reuse snippet in BlackboardResultWriter
209  //because for regex searches in UI we compress results by showing a content per regex once (even if multiple term hits)
210  //whereas in bb we write every hit per content separately
211  new BlackboardResultWriter(queryResults, queryRequest.getQuery().getKeywordList().getName()).execute();
212 
213  return true;
214  }
215 
223  Collection<KeywordHit> getOneHitPerObject(QueryResults queryResults) {
224  HashMap<Long, KeywordHit> hits = new HashMap<Long, KeywordHit>();
225  for (Keyword keyWord : queryResults.getKeywords()) {
226  for (KeywordHit hit : queryResults.getResults(keyWord)) {
227  // add hit with lowest SolrObjectID-Chunk-ID combination.
228  if (!hits.containsKey(hit.getSolrObjectId())) {
229  hits.put(hit.getSolrObjectId(), hit);
230  } else {
231  if (hit.getChunkId() < hits.get(hit.getSolrObjectId()).getChunkId()) {
232  hits.put(hit.getSolrObjectId(), hit);
233  }
234  }
235  }
236  }
237  return hits.values();
238  }
239 
249  private String getHighlightQuery(KeywordSearchQuery query, boolean literal_query, QueryResults queryResults, Content content) {
250  String highlightQueryEscaped;
251  if (literal_query) {
252  //literal, treat as non-regex, non-term component query
253  highlightQueryEscaped = query.getQueryString();
254  } else {
255  //construct a Solr query using aggregated terms to get highlighting
256  //the query is executed later on demand
257  StringBuilder highlightQuery = new StringBuilder();
258 
259  if (queryResults.getKeywords().size() == 1) {
260  //simple case, no need to process subqueries and do special escaping
261  Keyword term = queryResults.getKeywords().iterator().next();
262  highlightQuery.append(term.toString());
263  } else {
264  //find terms for this content hit
265  List<String> hitTerms = new ArrayList<>();
266  for (Keyword keyword : queryResults.getKeywords()) {
267  for (KeywordHit hit : queryResults.getResults(keyword)) {
268  if (hit.getContent().equals(content)) {
269  hitTerms.add(keyword.toString());
270  break; //go to next term
271  }
272  }
273  }
274 
275  final int lastTerm = hitTerms.size() - 1;
276  int curTerm = 0;
277  for (String term : hitTerms) {
278  //escape subqueries, they shouldn't be escaped again later
279  final String termS = KeywordSearchUtil.escapeLuceneQuery(term);
280  highlightQuery.append("\"");
281  highlightQuery.append(termS);
282  highlightQuery.append("\"");
283  if (lastTerm != curTerm) {
284  highlightQuery.append(" "); //acts as OR ||
285  //force HIGHLIGHT_FIELD_REGEX index and stored content
286  //in each term after first. First term taken care by HighlightedMatchesSource
287  highlightQuery.append(LuceneQuery.HIGHLIGHT_FIELD_REGEX).append(":");
288  }
289 
290  ++curTerm;
291  }
292  }
293  //String highlightQueryEscaped = KeywordSearchUtil.escapeLuceneQuery(highlightQuery.toString());
294  highlightQueryEscaped = highlightQuery.toString();
295  }
296 
297  return highlightQueryEscaped;
298  }
299 
300  @Override
301  protected Node createNodeForKey(KeyValueQueryContent key) {
302  final Content content = key.getContent();
303  final String queryStr = key.getQueryStr();;
304  QueryResults hits = key.getHits();
305 
306  Node kvNode = new KeyValueNode(key, Children.LEAF, Lookups.singleton(content));
307 
308  //wrap in KeywordSearchFilterNode for the markup content, might need to override FilterNode for more customization
309  // store the data in HighlightedMatchesSource so that it can be looked up (in content viewer)
310  HighlightedTextMarkup highlights = new HighlightedTextMarkup(key.solrObjectId, queryStr, !key.getQuery().isLiteral(), false, hits);
311  return new KeywordSearchFilterNode(highlights, kvNode);
312  }
313 
318  class KeyValueQueryContent extends KeyValue {
319 
320  private long solrObjectId;
321  private Content content;
322  private String queryStr;
323  private QueryResults hits;
324  private KeywordSearchQuery query;
325 
339  public KeyValueQueryContent(String name, Map<String, Object> map, int id, long solrObjectId, Content content, String queryStr, KeywordSearchQuery query, QueryResults hits) {
340  super(name, map, id);
341  this.solrObjectId = solrObjectId;
342  this.content = content;
343  this.queryStr = queryStr;
344  this.hits = hits;
345  this.query = query;
346  }
347 
348  Content getContent() {
349  return content;
350  }
351 
352  String getQueryStr() {
353  return queryStr;
354  }
355 
356  QueryResults getHits() {
357  return hits;
358  }
359 
360  KeywordSearchQuery getQuery() {
361  return query;
362  }
363  }
364 
369  static class BlackboardResultWriter extends SwingWorker<Object, Void> {
370 
371  private static List<BlackboardResultWriter> writers = new ArrayList<>();
372  //lock utilized to enqueue writers and limit execution to 1 at a time
373  private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy
374  //private static final Lock writerLock = rwLock.writeLock();
375  private ProgressHandle progress;
376  private KeywordSearchQuery query;
377  private String listName;
378  private QueryResults hits;
379  private Collection<BlackboardArtifact> newArtifacts = new ArrayList<>();
380  private static final int QUERY_DISPLAY_LEN = 40;
381 
382  BlackboardResultWriter(QueryResults hits, String listName) {
383  this.hits = hits;
384  this.query = hits.getQuery();
385  this.listName = listName;
386  }
387 
388  protected void finalizeWorker() {
389  deregisterWriter(this);
390 
391  EventQueue.invokeLater(new Runnable() {
392  @Override
393  public void run() {
394  progress.finish();
395  }
396  });
397  }
398 
399  @Override
400  protected Object doInBackground() throws Exception {
401  registerWriter(this); //register (synchronized on class) outside of writerLock to prevent deadlock
402  final String queryStr = query.getQueryString();
403  final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr;
404  //block until previous writer is done
405  //writerLock.lock();
406 
407  try {
408  progress = ProgressHandleFactory.createHandle(
409  NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), new Cancellable() {
410  @Override
411  public boolean cancel() {
412  return BlackboardResultWriter.this.cancel(true);
413  }
414  });
415 
416  // Create blackboard artifacts
417  newArtifacts = hits.writeAllHitsToBlackBoard(progress, null, this, false);
418  } finally {
419  finalizeWorker();
420  }
421 
422  return null;
423  }
424 
425  @Override
426  protected void done() {
427  try {
428  // test if any exceptions were thrown
429  get();
430  } catch (InterruptedException | ExecutionException ex) {
431  logger.log(Level.SEVERE, "Error querying ", ex); //NON-NLS
432  }
433  }
434 
435  private static synchronized void registerWriter(BlackboardResultWriter writer) {
436  writers.add(writer);
437  }
438 
439  private static synchronized void deregisterWriter(BlackboardResultWriter writer) {
440  writers.remove(writer);
441  }
442 
443  static synchronized void stopAllWriters() {
444  for (BlackboardResultWriter w : writers) {
445  w.cancel(true);
446  writers.remove(w);
447  }
448  }
449  }
450 }
static Logger getLogger(String name)
Definition: Logger.java:131

Copyright © 2012-2015 Basis Technology. Generated on: Mon Oct 19 2015
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.